Skip to contentSkip to conversation

ARM CCA Device Assignment support

RFC PATCH In review Aneesh Kumar K.V (Arm) 14 participants 188 messages Jul 28, 2025 – Oct 15, 2025 Page 1 of 2

Subsystems (13)
KVM PCI Crypto API arm64 +9 more subsystems
lore
Share & tools
lore

Aneesh Kumar K.V (Arm)

author maintainer
#0 lore raw

cover ARM CCA Device Assignment support

This patch series implements support for Device Assignment in the ARM CCA architecture. The code changes are based on Alp12 specification published here [1].

The code builds on the TSM framework patches posted at [2]. We add extension to that framework so that TSM is now used in both the host and the guest.

A DA workflow can be summarized as below:

Host:
step 1.
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind echo vfio-pci > /sys/bus/pci/devices/${DEVICE}/driver_override echo ${DEVICE} > /sys/bus/pci/drivers_probe

step 2.
echo 1 > /sys/bus/pci/devices/$DEVICE/tsm/connect

Now in the guest we follow the below steps

step 1:
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind

step 2: Move the device to TDISP LOCK state
echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/lock

step 3: Moves the device to TDISP RUN state echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/accept

step 4: Load the driver again.
echo ${DEVICE} > /sys/bus/pci/drivers_probe

I'm currently working against TSM v3, as TSM v4 lacks the necessary callbacks—bind, unbind, and guest_req—required for guest interactions.

The implementation also makes use of RHI interfaces that fall outside the current RHI specification [5]. Once the spec is finalized, the code will be aligned accordingly.

For now, I’ve retained validate_mmio and vdev_req exit handling within KVM. This will transition to a guest_req-based mechanism once the specification is updated.

At that point, all device assignment (DA)-specific VM exits will exit directly to the VMM, and will use the guest_req ioctl to handle exit reasons. As part of this change, the handlers realm_exit_vdev_req_handler, realm_exit_vdev_comm_handler, and realm_exit_dev_mem_map_handler will be removed.

Full patchset for the kernel and kvmtool can be found at [3] and [4]

[1] https://developer.arm.com/-/cdn-downloads/permalink/Architectures/Armv9/DEN0137_1.1-alp12.zip

[2] https://lore.kernel.org/all/[email protected]

[3] https://git.gitlab.arm.com/linux-arm/linux-cca.git cca/tdisp-upstream-post-v1
[4] https://git.gitlab.arm.com/linux-arm/kvmtool-cca.git cca/tdisp-upstream-post-v1
[5] https://developer.arm.com/documentation/den0148/latest/

Aneesh Kumar K.V (Arm) (35):
tsm: Add tsm_bind/unbind helpers
tsm: Move tsm core outside the host directory
tsm: Move dsm_dev from pci_tdi to pci_tsm
tsm: Support DMA Allocation from private memory
tsm: Don't overload connect
iommufd: Add and option to request for bar mapping with
IORESOURCE_EXCLUSIVE
iommufd/viommu: Add support to associate viommu with kvm instance iommufd/tsm: Add tsm_op iommufd ioctls
iommufd/vdevice: Add TSM Guest request uAPI iommufd/vdevice: Add TSM map ioctl
KVM: arm64: CCA: register host tsm platform device
coco: host: arm64: CCA host platform device driver
coco: host: arm64: Create a PDEV with rmm
coco: host: arm64: Device communication support
coco: host: arm64: Stop and destroy the physical device
coco: host: arm64: set_pubkey support
coco: host: arm64: Add support for creating a virtual device
coco: host: arm64: Add support for virtual device communication
coco: host: arm64: Stop and destroy virtual device
coco: guest: arm64: Update arm CCA guest driver
arm64: CCA: Register guest tsm callback
cca: guest: arm64: Realm device lock support
KVM: arm64: Add exit handler related to device assignment
coco: host: arm64: add RSI_RDEV_GET_INSTANCE_ID related exit handler
coco: host: arm64: Add support for device communication exit handler
coco: guest: arm64: Add support for collecting interface reports
coco: host: arm64: Add support for realm host interface (RHI)
coco: guest: arm64: Add support for fetching interface report and
certificate chain from host
coco: guest: arm64: Add support for guest initiated TDI bind/unbind
KVM: arm64: CCA: handle dev mem map/unmap
coco: guest: arm64: Validate mmio range found in the interface report
coco: guest: arm64: Add Realm device start and stop support
KVM: arm64: CCA: enable DA in realm create parameters
coco: guest: arm64: Add support for fetching device measurements
coco: guest: arm64: Add support for fetching device info

Lukas Wunner (3):
X.509: Make certificate parser public
X.509: Parse Subject Alternative Name in certificates X.509: Move certificate length retrieval into new helper

 arch/arm64/include/asm/kvm_rme.h              |   3 +
 arch/arm64/include/asm/mem_encrypt.h          |   6 +-
 arch/arm64/include/asm/rhi.h                  |  39 +
 arch/arm64/include/asm/rmi_cmds.h             | 173 ++++
 arch/arm64/include/asm/rmi_smc.h              | 210 ++++-
 arch/arm64/include/asm/rsi.h                  |   5 +-
 arch/arm64/include/asm/rsi_cmds.h             | 129 +++
 arch/arm64/include/asm/rsi_smc.h              |  60 ++
 arch/arm64/kernel/Makefile                    |   2 +-
 arch/arm64/kernel/rhi.c                       |  35 +
 arch/arm64/kernel/rsi.c                       |  26 +-
 arch/arm64/kvm/mmu.c                          |  45 +
 arch/arm64/kvm/rme-exit.c                     |  87 ++
 arch/arm64/kvm/rme.c                          | 208 ++++-
 arch/arm64/mm/mem_encrypt.c                   |  10 +
 crypto/asymmetric_keys/x509_cert_parser.c     |   9 +
 crypto/asymmetric_keys/x509_loader.c          |  38 +-
 crypto/asymmetric_keys/x509_parser.h          |  40 +-
 drivers/iommu/iommufd/device.c                |  54 ++
 drivers/iommu/iommufd/iommufd_private.h       |   7 +
 drivers/iommu/iommufd/main.c                  |  13 +
 drivers/iommu/iommufd/viommu.c                | 178 +++-
 drivers/pci/tsm.c                             | 229 ++++-
 drivers/vfio/pci/vfio_pci_core.c              |  20 +-
 drivers/virt/coco/Kconfig                     |   5 +-
 drivers/virt/coco/Makefile                    |   7 +-
 drivers/virt/coco/arm-cca-guest/Kconfig       |  10 +-
 drivers/virt/coco/arm-cca-guest/Makefile      |   3 +
 .../{arm-cca-guest.c => arm-cca.c}            | 175 +++-
 drivers/virt/coco/arm-cca-guest/rsi-da.c      | 576 ++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h      |  73 ++
 drivers/virt/coco/arm-cca-host/Kconfig        |  17 +
 drivers/virt/coco/arm-cca-host/Makefile       |   5 +
 drivers/virt/coco/arm-cca-host/arm-cca.c      | 384 ++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.c       | 857 ++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h       | 108 +++
 drivers/virt/coco/host/Kconfig                |   6 -
 drivers/virt/coco/host/Makefile               |   6 -
 drivers/virt/coco/{host => }/tsm-core.c       |  27 +
 include/keys/asymmetric-type.h                |   2 +
 include/keys/x509-parser.h                    |  55 ++
 include/linux/device.h                        |   1 +
 include/linux/iommufd.h                       |   4 +
 include/linux/kvm_host.h                      |   1 +
 include/linux/pci-tsm.h                       |  37 +-
 include/linux/swiotlb.h                       |   4 +
 include/linux/tsm.h                           |  29 +
 include/uapi/linux/iommufd.h                  |  69 ++
 48 files changed, 3887 insertions(+), 200 deletions(-)
 create mode 100644 arch/arm64/include/asm/rhi.h
 create mode 100644 arch/arm64/kernel/rhi.c
 rename drivers/virt/coco/arm-cca-guest/{arm-cca-guest.c => arm-cca.c} (62%)
 create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.c
 create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.h
 create mode 100644 drivers/virt/coco/arm-cca-host/Kconfig
 create mode 100644 drivers/virt/coco/arm-cca-host/Makefile
 create mode 100644 drivers/virt/coco/arm-cca-host/arm-cca.c
 create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.c
 create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.h
 delete mode 100644 drivers/virt/coco/host/Kconfig
 delete mode 100644 drivers/virt/coco/host/Makefile
 rename drivers/virt/coco/{host => }/tsm-core.c (85%)
 create mode 100644 include/keys/x509-parser.h
Signature · 1 line
2.43.0
author maintainer
#1 lore raw

RFC PATCH 1/38 tsm: Add tsm_bind/unbind helpers

This will be later used by iommufd to bind a tdi to guest.

---
 drivers/virt/coco/host/tsm-core.c | 18 ++++++++++++++++++
 include/linux/tsm.h               |  3 +++
 2 files changed, 21 insertions(+)
drivers/virt/coco/host/tsm-core.c +18 -0
index bd9e09b07412..0a7c9aa46c56 100644
@@ -116,6 +116,24 @@ void tsm_ide_stream_unregister(struct pci_ide *ide)
 }
 EXPORT_SYMBOL_GPL(tsm_ide_stream_unregister);
 
+int tsm_bind(struct device *dev, struct kvm *kvm, u64 tdi_id)
+{
+	if (!dev_is_pci(dev))
+		return -EINVAL;
+
+	return pci_tsm_bind(to_pci_dev(dev), kvm, tdi_id);
+}
+EXPORT_SYMBOL_GPL(tsm_bind);
+
+int tsm_unbind(struct device *dev)
+{
+	if (!dev_is_pci(dev))
+		return -EINVAL;
+
+	return pci_tsm_unbind(to_pci_dev(dev));
+}
+EXPORT_SYMBOL_GPL(tsm_unbind);
+
 static void tsm_release(struct device *dev)
 {
 	struct tsm_core_dev *core = container_of(dev, typeof(*core), dev);
include/linux/tsm.h +3 -0
index 915c4c8b061b..0aab8d037e71 100644
@@ -118,6 +118,9 @@ struct tsm_core_dev *tsm_register(struct device *parent,
 void tsm_unregister(struct tsm_core_dev *tsm_core);
 struct pci_dev;
 struct pci_ide;
+struct kvm;
 int tsm_ide_stream_register(struct pci_dev *pdev, struct pci_ide *ide);
 void tsm_ide_stream_unregister(struct pci_ide *ide);
+int tsm_bind(struct device *dev, struct kvm *kvm, u64 tdi_id);
+int tsm_unbind(struct device *dev);
 #endif /* __TSM_H */
Signature · 1 line
2.43.0
author maintainer
#2 lore raw

RFC PATCH 2/38 tsm: Move tsm core outside the host directory

In reply to Aneesh Kumar K.V (Arm)

A later patch will add guest changes that will also use the same infrastructure.

---
 drivers/virt/coco/Kconfig               | 3 ++-
 drivers/virt/coco/Makefile              | 6 ++++--
 drivers/virt/coco/host/Kconfig          | 6 ------
 drivers/virt/coco/host/Makefile         | 6 ------
 drivers/virt/coco/{host => }/tsm-core.c | 0
 5 files changed, 6 insertions(+), 15 deletions(-)
 delete mode 100644 drivers/virt/coco/host/Kconfig
 delete mode 100644 drivers/virt/coco/host/Makefile
 rename drivers/virt/coco/{host => }/tsm-core.c (100%)
drivers/virt/coco/Kconfig +2 -1
index 14e7cf145d85..57248b088545 100644
@@ -15,4 +15,5 @@ source "drivers/virt/coco/arm-cca-guest/Kconfig"
 
 source "drivers/virt/coco/guest/Kconfig"
 
-source "drivers/virt/coco/host/Kconfig"
+config TSM
+	tristate
drivers/virt/coco/Makefile +4 -2
index 73f1b7bc5b11..04e124b2d7cf 100644
@@ -2,10 +2,12 @@
 #
 # Confidential computing related collateral
 #
+
 obj-$(CONFIG_EFI_SECRET)	+= efi_secret/
 obj-$(CONFIG_ARM_PKVM_GUEST)	+= pkvm-guest/
 obj-$(CONFIG_SEV_GUEST)		+= sev-guest/
 obj-$(CONFIG_INTEL_TDX_GUEST)	+= tdx-guest/
 obj-$(CONFIG_ARM_CCA_GUEST)	+= arm-cca-guest/
-obj-$(CONFIG_TSM_REPORTS)	+= guest/
-obj-y				+= host/
+
+obj-$(CONFIG_TSM) 		+= tsm-core.o
+obj-y				+= guest/
drivers/virt/coco/host/Kconfig deleted +0 -6
deleted file mode 100644
index 4fbc6ef34f12..000000000000
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# TSM (TEE Security Manager) Common infrastructure and host drivers
-#
-config TSM
-	tristate
drivers/virt/coco/host/Makefile deleted +0 -6
deleted file mode 100644
index be0aba6007cd..000000000000
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# TSM (TEE Security Manager) Common infrastructure and host drivers
-
-obj-$(CONFIG_TSM) += tsm.o
-tsm-y := tsm-core.o
drivers/virt/coco/tsm-core.c renamed from drivers/virt/coco/host/tsm-core.c +0 -0
similarity index 100%
rename from drivers/virt/coco/host/tsm-core.c
rename to drivers/virt/coco/tsm-core.c
Signature · 1 line
2.43.0
author maintainer
#3 lore raw

RFC PATCH 3/38 tsm: Move dsm_dev from pci_tdi to pci_tsm

In reply to Aneesh Kumar K.V (Arm)
---
 drivers/pci/tsm.c       | 72 ++++++++++++++++++++++++-----------------
 include/linux/pci-tsm.h |  4 +--
 2 files changed, 45 insertions(+), 31 deletions(-)
drivers/pci/tsm.c +43 -29
index 794de2f258c3..e4a3b5b37939 100644
@@ -415,15 +415,55 @@ static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev)
 	return PCI_TSM_INVALID;
 }
 
+/* lookup the Device Security Manager (DSM) pf0 for @pdev */
+static struct pci_dev *dsm_dev_get(struct pci_dev *pdev)
+{
+	struct pci_dev *uport_pf0;
+
+	struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev);
+	if (!pf0)
+		return NULL;
+
+	if (pf0 == pdev)
+		return no_free_ptr(pf0);
+
+	/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */
+	if (pf0->tsm && pf0->tsm->type == PCI_TSM_PF0)
+		return no_free_ptr(pf0);
+
+	/*
+	 * For cases where a switch may be hosting TDISP services on
+	 * behalf of downstream devices, check the first usptream port
+	 * relative to this endpoint.
+	 */
+	if (!pdev->dev.parent || !pdev->dev.parent->parent)
+		return NULL;
+
+	uport_pf0 = to_pci_dev(pdev->dev.parent->parent);
+	if (!uport_pf0->tsm)
+		return NULL;
+	return pci_dev_get(uport_pf0);
+}
+
 /**
  * pci_tsm_initialize() - base 'struct pci_tsm' initialization
  * @pdev: The PCI device
  * @tsm: context to initialize
  */
-void pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm)
+int pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm)
 {
+	struct pci_dev *dsm_dev __free(pci_dev_put) = dsm_dev_get(pdev);
+	if (!dsm_dev)
+		return -EINVAL;
+
 	tsm->type = pci_tsm_type(pdev);
 	tsm->pdev = pdev;
+	/*
+	 * No reference needed because when we destroy
+	 * dsm_dev all the tdis get destroyed before that.
+	 */
+	tsm->dsm_dev = dsm_dev;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pci_tsm_initialize);
 
@@ -447,7 +487,8 @@ int pci_tsm_pf0_initialize(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm)
 	}
 
 	tsm->state = PCI_TSM_INIT;
-	pci_tsm_initialize(pdev, &tsm->tsm);
+	if (pci_tsm_initialize(pdev, &tsm->tsm))
+		return -ENODEV;
 
 	return 0;
 }
@@ -612,32 +653,6 @@ int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type,
 }
 EXPORT_SYMBOL_GPL(pci_tsm_doe_transfer);
 
-/* lookup the Device Security Manager (DSM) pf0 for @pdev */
-static struct pci_dev *dsm_dev_get(struct pci_dev *pdev)
-{
-	struct pci_dev *uport_pf0;
-
-	struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev);
-	if (!pf0)
-		return NULL;
-
-	/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */
-	if (pf0->tsm && pf0->tsm->type == PCI_TSM_PF0)
-		return no_free_ptr(pf0);
-
-	/*
-	 * For cases where a switch may be hosting TDISP services on
-	 * behalf of downstream devices, check the first usptream port
-	 * relative to this endpoint.
-	 */
-	if (!pdev->dev.parent || !pdev->dev.parent->parent)
-		return NULL;
-
-	uport_pf0 = to_pci_dev(pdev->dev.parent->parent);
-	if (!uport_pf0->tsm)
-		return NULL;
-	return pci_dev_get(uport_pf0);
-}
 
 /* Only implement non-interruptible lock for now */
 static struct mutex *tdi_ops_lock(struct pci_dev *pf0_dev)
@@ -695,7 +710,6 @@ int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id)
 		return -ENXIO;
 
 	tdi->pdev = pdev;
-	tdi->dsm_dev = dsm_dev;
 	tdi->kvm = kvm;
 	pdev->tsm->tdi = tdi;
 
include/linux/pci-tsm.h +2 -2
index 1920ca591a42..0d4303726b25 100644
@@ -38,7 +38,6 @@ enum pci_tsm_type {
  */
 struct pci_tdi {
 	struct pci_dev *pdev;
-	struct pci_dev *dsm_dev;
 	struct kvm *kvm;
 };
 
@@ -56,6 +55,7 @@ struct pci_tdi {
  */
 struct pci_tsm {
 	struct pci_dev *pdev;
+	struct pci_dev *dsm_dev;
 	enum pci_tsm_type type;
 	struct pci_tdi *tdi;
 };
@@ -173,7 +173,7 @@ void pci_tsm_core_unregister(const struct pci_tsm_ops *ops);
 int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type,
 			 const void *req, size_t req_sz, void *resp,
 			 size_t resp_sz);
-void pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm);
+int pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm);
 int pci_tsm_pf0_initialize(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm);
 int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id);
 int pci_tsm_unbind(struct pci_dev *pdev);
Signature · 1 line
2.43.0
author maintainer
#6 lore raw

RFC PATCH 4/38 tsm: Support DMA Allocation from private memory

In reply to Aneesh Kumar K.V (Arm)

Currently, we enforce the use of bounce buffers to ensure that memory accessed by non-secure devices is explicitly shared with the host [1]. However, for secure devices, this approach must be avoided.

To achieve this, we introduce a device flag that controls whether a bounce buffer allocation is required for the device. Additionally, this flag is used to manage the top IPA bit assignment for setting up protected/unprotected IPA aliases.

[1] commit fbf979a01375 ("arm64: Enforce bounce buffers for realm DMA")

based on changes from Alexey Kardashevskiy <[email protected]>

---
 arch/arm64/include/asm/mem_encrypt.h |  6 +-----
 arch/arm64/mm/mem_encrypt.c          | 10 ++++++++++
 drivers/pci/tsm.c                    |  6 ++++++
 include/linux/device.h               |  1 +
 include/linux/swiotlb.h              |  4 ++++
 5 files changed, 22 insertions(+), 5 deletions(-)
arch/arm64/include/asm/mem_encrypt.h +1 -5
index 314b2b52025f..d77c10cd5b79 100644
@@ -15,14 +15,10 @@ int arm64_mem_crypt_ops_register(const struct arm64_mem_crypt_ops *ops);
 
 int set_memory_encrypted(unsigned long addr, int numpages);
 int set_memory_decrypted(unsigned long addr, int numpages);
+bool force_dma_unencrypted(struct device *dev);
 
 int realm_register_memory_enc_ops(void);
 
-static inline bool force_dma_unencrypted(struct device *dev)
-{
-	return is_realm_world();
-}
-
 /*
  * For Arm CCA guests, canonical addresses are "encrypted", so no changes
  * required for dma_addr_encrypted().
arch/arm64/mm/mem_encrypt.c +10 -0
index ee3c0ab04384..279696a8af3f 100644
@@ -17,6 +17,7 @@
 #include <linux/compiler.h>
 #include <linux/err.h>
 #include <linux/mm.h>
+#include <linux/device.h>
 
 #include <asm/mem_encrypt.h>
 
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
 	return crypt_ops->decrypt(addr, numpages);
 }
 EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;
+
+	return is_realm_world();
+}
+EXPORT_SYMBOL_GPL(force_dma_unencrypted);
drivers/pci/tsm.c +6 -0
index e4a3b5b37939..60f50d57a725 100644
@@ -120,6 +120,7 @@ static int pci_tsm_disconnect(struct pci_dev *pdev)
 
 	tsm_ops->disconnect(pdev);
 	tsm->state = PCI_TSM_INIT;
+	pdev->dev.tdi_enabled = false;
 
 	return 0;
 }
@@ -199,6 +200,8 @@ static int pci_tsm_accept(struct pci_dev *pdev)
 	if (rc)
 		return rc;
 	tsm->state = PCI_TSM_ACCEPT;
+	pdev->dev.tdi_enabled = true;
+
 	return 0;
 }
 
@@ -557,6 +560,9 @@ static void __pci_tsm_init(struct pci_dev *pdev)
 	default:
 		break;
 	}
+
+	/* FIXME!! should this be default true and switch to false for TEE capable device */
+	pdev->dev.tdi_enabled = false;
 }
 
 void pci_tsm_init(struct pci_dev *pdev)
include/linux/device.h +1 -0
index 4940db137fff..d62e0dd9d8ee 100644
@@ -688,6 +688,7 @@ struct device {
 #ifdef CONFIG_IOMMU_DMA
 	bool			dma_iommu:1;
 #endif
+	bool			tdi_enabled:1;
 };
 
 /**
include/linux/swiotlb.h +4 -0
index 3dae0f592063..61e7cff7768b 100644
@@ -173,6 +173,10 @@ static inline bool is_swiotlb_force_bounce(struct device *dev)
 {
 	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
 
+	if (dev->tdi_enabled) {
+		dev_warn_once(dev, "(TIO) Disable SWIOTLB");
+		return false;
+	}
 	return mem && mem->force_bounce;
 }
 
Signature · 1 line
2.43.0
author maintainer
#26 lore raw

RFC PATCH 5/38 tsm: Don't overload connect

In reply to Aneesh Kumar K.V (Arm)

We need separate handling in the guest while destroying the device. Hence switch to new callback lock and unlock. Hide the connect sysfs file in guest.

guest sysfs will now looks like
ls -al /sys/bus/pci/devices/0000:02:00.0/tsm/
total 0

drwxr-xr-x    2 root     root             0 Jan  1 00:00 .
drwxr-xr-x    7 root     root             0 Jan  1 00:00 ..
-rw-r--r--    1 root     root          4096 Jan  1 00:00 accept
-rw-r--r--    1 root     root          4096 Jan  1 00:00 lock
---
 drivers/pci/tsm.c       | 136 ++++++++++++++++++++++++++++++++++------
 include/linux/pci-tsm.h |   3 +
 2 files changed, 121 insertions(+), 18 deletions(-)
drivers/pci/tsm.c +118 -18
index 60f50d57a725..80607082b7f0 100644
@@ -125,18 +125,6 @@ static int pci_tsm_disconnect(struct pci_dev *pdev)
 	return 0;
 }
 
-/*
- * TDISP locked state temporarily makes the device inaccessible, do not
- * surprise live attached drivers
- */
-static int __driver_idle_connect(struct pci_dev *pdev)
-{
-	guard(device)(&pdev->dev);
-	if (pdev->dev.driver)
-		return -EBUSY;
-	return tsm_ops->connect(pdev);
-}
-
 /*
  * When the registered ops support accept it indicates that this is a
  * TVM-side (guest) TSM operations structure. In this mode ->connect()
@@ -162,10 +150,7 @@ static int pci_tsm_connect(struct pci_dev *pdev)
 	if (tsm->state >= PCI_TSM_CONNECT)
 		return 0;
 
-	if (tvm_mode())
-		rc = __driver_idle_connect(pdev);
-	else
-		rc = tsm_ops->connect(pdev);
+	rc = tsm_ops->connect(pdev);
 	if (rc)
 		return rc;
 	tsm->state = PCI_TSM_CONNECT;
@@ -299,6 +284,99 @@ static ssize_t accept_show(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(accept);
 
+static int pci_tsm_unlock(struct pci_dev *pdev)
+{
+	struct pci_tsm_pf0 *tsm = to_pci_tsm_pf0(pdev->tsm);
+
+	struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm);
+	if (!lock)
+		return -EINTR;
+
+	if (tsm->state < PCI_TSM_INIT)
+		return -ENXIO;
+	if (tsm->state < PCI_TSM_LOCK)
+		return 0;
+
+	tsm_ops->unlock(pdev);
+	tsm->state = PCI_TSM_INIT;
+	pdev->dev.tdi_enabled = false;
+
+	return 0;
+}
+
+/*
+ * TDISP locked state temporarily makes the device inaccessible, do not
+ * surprise live attached drivers
+ */
+static int __driver_idle_lock(struct pci_dev *pdev)
+{
+	guard(device)(&pdev->dev);
+	if (pdev->dev.driver)
+		return -EBUSY;
+	return tsm_ops->lock(pdev);
+}
+
+static int pci_tsm_lock(struct pci_dev *pdev)
+{
+	struct pci_tsm_pf0 *tsm = to_pci_tsm_pf0(pdev->tsm);
+	int rc;
+
+	struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm);
+	if (!lock)
+		return -EINTR;
+
+	if (tsm->state < PCI_TSM_INIT)
+		return -ENXIO;
+
+	rc = __driver_idle_lock(pdev);
+	if (rc)
+		return rc;
+	tsm->state = PCI_TSM_CONNECT;
+	return 0;
+}
+
+static ssize_t lock_store(struct device *dev, struct device_attribute *attr,
+			  const char *buf, size_t len)
+{
+	int rc;
+	bool connect;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	rc = kstrtobool(buf, &connect);
+	if (rc)
+		return rc;
+
+	struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock();
+	if (!lock)
+		return -EINTR;
+
+	if (connect)
+		rc = pci_tsm_lock(pdev);
+	else
+		rc = pci_tsm_unlock(pdev);
+	if (rc)
+		return rc;
+	return len;
+}
+
+static ssize_t lock_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_tsm_pf0 *tsm;
+
+	struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock();
+	if (!lock)
+		return -EINTR;
+
+	if (!pdev->tsm)
+		return -ENXIO;
+
+	tsm = to_pci_tsm_pf0(pdev->tsm);
+	return sysfs_emit(buf, "%d\n", tsm->state >= PCI_TSM_LOCK);
+}
+static DEVICE_ATTR_RW(lock);
+
 static umode_t pci_tsm_pf0_attr_visible(struct kobject *kobj,
 					struct attribute *a, int n)
 {
@@ -306,6 +384,11 @@ static umode_t pci_tsm_pf0_attr_visible(struct kobject *kobj,
 		/* Host context, filter out guest only attributes */
 		if (a == &dev_attr_accept.attr)
 			return 0;
+		if (a == &dev_attr_lock.attr)
+			return 0;
+	} else {
+		if (a == &dev_attr_connect.attr)
+			return 0;
 	}
 
 	return a->mode;
@@ -325,6 +408,7 @@ DEFINE_SYSFS_GROUP_VISIBLE(pci_tsm_pf0);
 static struct attribute *pci_tsm_pf0_attrs[] = {
 	&dev_attr_connect.attr,
 	&dev_attr_accept.attr,
+	&dev_attr_lock.attr,
 	NULL
 };
 
@@ -537,7 +621,8 @@ static void pci_tsm_pf0_init(struct pci_dev *pdev)
 		return;
 
 	pdev->tsm = no_free_ptr(pci_tsm);
-	sysfs_update_group(&pdev->dev.kobj, &pci_tsm_auth_attr_group);
+	if (!tvm_mode())
+		sysfs_update_group(&pdev->dev.kobj, &pci_tsm_auth_attr_group);
 	sysfs_update_group(&pdev->dev.kobj, &pci_tsm_pf0_attr_group);
 	if (pci_tsm_owner_attr_group)
 		sysfs_merge_group(&pdev->dev.kobj, pci_tsm_owner_attr_group);
@@ -602,6 +687,19 @@ static void pci_tsm_pf0_destroy(struct pci_dev *pdev)
 	__pci_tsm_pf0_destroy(tsm);
 }
 
+static void pci_tsm_guest_destroy(struct pci_dev *pdev)
+{
+	struct pci_tsm_pf0 *tsm = to_pci_tsm_pf0(pdev->tsm);
+
+	if (tsm->state > PCI_TSM_INIT)
+		pci_tsm_unlock(pdev);
+	pdev->tsm = NULL;
+	if (pci_tsm_owner_attr_group)
+		sysfs_unmerge_group(&pdev->dev.kobj, pci_tsm_owner_attr_group);
+	sysfs_update_group(&pdev->dev.kobj, &pci_tsm_pf0_attr_group);
+	__pci_tsm_pf0_destroy(tsm);
+}
+
 static void __pci_tsm_destroy(struct pci_dev *pdev)
 {
 	struct pci_tsm *pci_tsm = pdev->tsm;
@@ -611,7 +709,9 @@ static void __pci_tsm_destroy(struct pci_dev *pdev)
 
 	lockdep_assert_held_write(&pci_tsm_rwsem);
 
-	if (is_pci_tsm_pf0(pdev)) {
+	if (tvm_mode()) {
+		pci_tsm_guest_destroy(pdev);
+	} else if (is_pci_tsm_pf0(pdev)) {
 		pci_tsm_pf0_destroy(pdev);
 	} else {
 		__pci_tsm_unbind(pdev);
include/linux/pci-tsm.h +3 -0
index 0d4303726b25..7639e7963681 100644
@@ -11,6 +11,7 @@ enum pci_tsm_state {
 	PCI_TSM_ERR = -1,
 	PCI_TSM_INIT,
 	PCI_TSM_CONNECT,
+	PCI_TSM_LOCK,
 	PCI_TSM_ACCEPT,
 };
 
@@ -153,6 +154,8 @@ struct pci_tsm_ops {
 	void (*remove)(struct pci_tsm *tsm);
 	int (*connect)(struct pci_dev *pdev);
 	void (*disconnect)(struct pci_dev *pdev);
+	int (*lock)(struct pci_dev *pdev);
+	void (*unlock)(struct pci_dev *pdev);
 	struct pci_tdi *(*bind)(struct pci_dev *pdev, struct pci_dev *pf0_dev,
 				struct kvm *kvm, u64 tdi_id);
 	void (*unbind)(struct pci_tdi *tdi);
Signature · 1 line
2.43.0
author maintainer
#28 lore raw

RFC PATCH 6/38 iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE

In reply to Aneesh Kumar K.V (Arm)
---
 drivers/iommu/iommufd/device.c          | 54 +++++++++++++++++++++++++
 drivers/iommu/iommufd/iommufd_private.h |  4 ++
 drivers/iommu/iommufd/main.c            |  3 ++
 drivers/vfio/pci/vfio_pci_core.c        |  9 ++++-
 include/linux/iommufd.h                 |  1 +
 include/uapi/linux/iommufd.h            |  1 +
 6 files changed, 71 insertions(+), 1 deletion(-)
drivers/iommu/iommufd/device.c +54 -0
index 65fbd098f9e9..3bd6972836a1 100644
@@ -1660,3 +1660,57 @@ int iommufd_get_hw_info(struct iommufd_ucmd *ucmd)
 	iommufd_put_object(ucmd->ictx, &idev->obj);
 	return rc;
 }
+
+static int iommufd_device_option_exclusive_ranges(struct iommu_option *cmd,
+					  struct iommufd_device *idev)
+{
+	if (cmd->op == IOMMU_OPTION_OP_GET) {
+		cmd->val64 = idev->flags & IOMMUFD_DEVICE_EXCLUSIVE_RANGE;
+		return 0;
+	}
+
+	if (cmd->op == IOMMU_OPTION_OP_SET) {
+		if (cmd->val64 == 0) {
+			idev->flags &= ~IOMMUFD_DEVICE_EXCLUSIVE_RANGE;
+			return 0;
+		} else if (cmd->val64 & IOMMUFD_DEVICE_EXCLUSIVE_RANGE) {
+			idev->flags |= IOMMUFD_DEVICE_EXCLUSIVE_RANGE;
+			return 0;
+		}
+		return -EINVAL;
+	}
+	return -EOPNOTSUPP;
+}
+bool iommufd_device_need_exclusive_range(struct iommufd_device *idev)
+{
+	return !!(idev->flags & IOMMUFD_DEVICE_EXCLUSIVE_RANGE);
+}
+
+int iommufd_device_option(struct iommufd_ucmd *ucmd)
+{
+	struct iommu_option *cmd = ucmd->cmd;
+	struct iommufd_device *idev;
+	int rc = 0;
+
+	if (cmd->__reserved)
+		return -EOPNOTSUPP;
+
+	idev = iommufd_get_device(ucmd, cmd->object_id);
+	if (IS_ERR(idev))
+		return PTR_ERR(idev);
+
+	mutex_lock(&idev->igroup->lock);
+
+
+	switch (cmd->option_id) {
+	case IOMMU_OPTION_EXCLUSIVE_RANGES:
+		rc = iommufd_device_option_exclusive_ranges(cmd, idev);
+		break;
+	default:
+		rc = -EOPNOTSUPP;
+	}
+
+	mutex_unlock(&idev->igroup->lock);
+	iommufd_put_object(ucmd->ictx, &idev->obj);
+	return rc;
+}
drivers/iommu/iommufd/iommufd_private.h +4 -0
index 0da2a81eedfa..fce68714c80f 100644
@@ -346,6 +346,7 @@ int iommufd_ioas_change_process(struct iommufd_ucmd *ucmd);
 int iommufd_ioas_copy(struct iommufd_ucmd *ucmd);
 int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd);
 int iommufd_ioas_option(struct iommufd_ucmd *ucmd);
+int iommufd_device_option(struct iommufd_ucmd *ucmd);
 int iommufd_option_rlimit_mode(struct iommu_option *cmd,
 			       struct iommufd_ctx *ictx);
 
@@ -489,10 +490,13 @@ struct iommufd_device {
 	/* always the physical device */
 	struct device *dev;
 	bool enforce_cache_coherency;
+	unsigned long flags;
 	struct iommufd_vdevice *vdev;
 	bool destroying;
 };
 
+#define IOMMUFD_DEVICE_EXCLUSIVE_RANGE		BIT(0)
+
 static inline struct iommufd_device *
 iommufd_get_device(struct iommufd_ucmd *ucmd, u32 id)
 {
drivers/iommu/iommufd/main.c +3 -0
index 15af7ced0501..89830da8b418 100644
@@ -376,6 +376,9 @@ static int iommufd_option(struct iommufd_ucmd *ucmd)
 	case IOMMU_OPTION_HUGE_PAGES:
 		rc = iommufd_ioas_option(ucmd);
 		break;
+	case IOMMU_OPTION_EXCLUSIVE_RANGES:
+		rc = iommufd_device_option(ucmd);
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
drivers/vfio/pci/vfio_pci_core.c +8 -1
index 6328c3a05bcd..bee3cf3226e9 100644
@@ -1753,8 +1753,15 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
 	 * we need to request the region and the barmap tracks that.
 	 */
 	if (!vdev->barmap[index]) {
-		ret = pci_request_selected_regions(pdev,
+
+		if (core_vdev->iommufd_device &&
+		    iommufd_device_need_exclusive_range(core_vdev->iommufd_device))
+			ret = pci_request_selected_regions_exclusive(pdev,
+							1 << index, "vfio-pci");
+		else
+			ret = pci_request_selected_regions(pdev,
 						   1 << index, "vfio-pci");
+
 		if (ret)
 			return ret;
 
include/linux/iommufd.h +1 -0
index 6e7efe83bc5d..55ae02581f9b 100644
@@ -70,6 +70,7 @@ void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid);
 
 struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev);
 u32 iommufd_device_to_id(struct iommufd_device *idev);
+bool iommufd_device_need_exclusive_range(struct iommufd_device *idev);
 
 struct iommufd_access_ops {
 	u8 needs_pin_pages : 1;
include/uapi/linux/iommufd.h +1 -0
index c218c89e0e2e..548d4b5afcd4 100644
@@ -310,6 +310,7 @@ struct iommu_ioas_unmap {
 enum iommufd_option {
 	IOMMU_OPTION_RLIMIT_MODE = 0,
 	IOMMU_OPTION_HUGE_PAGES = 1,
+	IOMMU_OPTION_EXCLUSIVE_RANGES = 2,
 };
 
 /**
Signature · 1 line
2.43.0
author maintainer
#38 lore raw

RFC PATCH 7/38 iommufd/viommu: Add support to associate viommu with kvm instance

In reply to Aneesh Kumar K.V (Arm)

The associated kvm instance will be used in later patch by iommufd to bind a tdi to kvm.

---
 drivers/iommu/iommufd/viommu.c | 45 +++++++++++++++++++++++++++++++++-
 include/linux/iommufd.h        |  3 +++
 include/uapi/linux/iommufd.h   | 12 +++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)
drivers/iommu/iommufd/viommu.c +44 -1
index 2ca5809b238b..59f1e1176f7f 100644
@@ -2,6 +2,36 @@
 /* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
  */
 #include "iommufd_private.h"
+#include "linux/tsm.h"
+
+#if IS_ENABLED(CONFIG_KVM)
+#include <linux/kvm_host.h>
+
+static int viommu_get_kvm(struct iommufd_viommu *viommu, int kvm_vm_fd)
+{
+	int rc = -EBADF;
+	struct file *filp;
+
+	filp = fget(kvm_vm_fd);
+
+	if (!file_is_kvm(filp))
+		goto err_out;
+
+	/* hold the kvm reference via file descriptor */
+	viommu->kvm_filp = filp;
+	return 0;
+err_out:
+	viommu->kvm_filp = NULL;
+	fput(filp);
+	return rc;
+}
+
+static void viommu_put_kvm(struct iommufd_viommu *viommu)
+{
+	fput(viommu->kvm_filp);
+	viommu->kvm_filp = NULL;
+}
+#endif
 
 void iommufd_viommu_destroy(struct iommufd_object *obj)
 {
@@ -12,6 +42,8 @@ void iommufd_viommu_destroy(struct iommufd_object *obj)
 		viommu->ops->destroy(viommu);
 	refcount_dec(&viommu->hwpt->common.obj.users);
 	xa_destroy(&viommu->vdevs);
+
+	viommu_put_kvm(viommu);
 }
 
 int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
@@ -29,7 +61,9 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	size_t viommu_size;
 	int rc;
 
-	if (cmd->flags || cmd->type == IOMMU_VIOMMU_TYPE_DEFAULT)
+	if (cmd->flags & ~IOMMU_VIOMMU_KVM_FD)
+		return -EOPNOTSUPP;
+	if (cmd->type == IOMMU_VIOMMU_TYPE_DEFAULT)
 		return -EOPNOTSUPP;
 
 	idev = iommufd_get_device(ucmd, cmd->dev_id);
@@ -100,8 +134,17 @@ int iommufd_viommu_alloc_ioctl(struct iommufd_ucmd *ucmd)
 		goto out_put_hwpt;
 	}
 
+	/* get the kvm details if specified. */
+	if (cmd->flags & IOMMU_VIOMMU_KVM_FD) {
+		rc = viommu_get_kvm(viommu, cmd->kvm_vm_fd);
+		if (rc)
+			goto out_put_hwpt;
+	}
+
 	cmd->out_viommu_id = viommu->obj.id;
 	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
+	if (rc)
+		viommu_put_kvm(viommu);
 
 out_put_hwpt:
 	iommufd_put_object(ucmd->ictx, &hwpt_paging->common.obj);
include/linux/iommufd.h +3 -0
index 55ae02581f9b..b7617ba7a536 100644
@@ -12,6 +12,7 @@
 #include <linux/refcount.h>
 #include <linux/types.h>
 #include <linux/xarray.h>
+#include <linux/file.h>
 #include <uapi/linux/iommufd.h>
 
 struct device;
@@ -58,6 +59,7 @@ struct iommufd_object {
 	unsigned int id;
 };
 
+struct kvm;
 struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
 					   struct device *dev, u32 *id);
 void iommufd_device_unbind(struct iommufd_device *idev);
@@ -102,6 +104,7 @@ struct iommufd_viommu {
 	struct iommufd_ctx *ictx;
 	struct iommu_device *iommu_dev;
 	struct iommufd_hwpt_paging *hwpt;
+	struct file *kvm_filp;
 
 	const struct iommufd_viommu_ops *ops;
 
include/uapi/linux/iommufd.h +12 -0
index 548d4b5afcd4..9014c61a97d4 100644
@@ -1023,6 +1023,17 @@ struct iommu_viommu_tegra241_cmdqv {
 	__aligned_u64 out_vintf_mmap_length;
 };
 
+/**
+ * define IOMMU_VIOMMU_KVM_FD - Flag indicating a valid KVM VM file descriptor
+ *
+ * This flag must be set when allocating a viommu instance that will be
+ * associated with a specific KVM VM. When allocating a viommu instance for a
+ * KVM VM, this flag must be set to inform the initialization logic that
+ * @iommu_viommu_alloc::kvm_vm_fd is properly initialized. If this flag is not
+ * provided but @iommu_viommu_alloc::kvm_vm_fd field will be ignored.
+ */
+#define IOMMU_VIOMMU_KVM_FD	BIT(0)
+
 /**
  * struct iommu_viommu_alloc - ioctl(IOMMU_VIOMMU_ALLOC)
  * @size: sizeof(struct iommu_viommu_alloc)
@@ -1057,6 +1068,7 @@ struct iommu_viommu_alloc {
 	__u32 data_len;
 	__u32 __reserved;
 	__aligned_u64 data_uptr;
+	__u32 kvm_vm_fd;
 };
 #define IOMMU_VIOMMU_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VIOMMU_ALLOC)
 
Signature · 1 line
2.43.0
author maintainer
#43 lore raw

RFC PATCH 8/38 iommufd/tsm: Add tsm_op iommufd ioctls

In reply to Aneesh Kumar K.V (Arm)

Add operations bind and unbind used to bind a TDI to the secure guest.

---
 drivers/iommu/iommufd/iommufd_private.h |  1 +
 drivers/iommu/iommufd/main.c            |  3 ++
 drivers/iommu/iommufd/viommu.c          | 50 +++++++++++++++++++++++++
 drivers/vfio/pci/vfio_pci_core.c        | 10 +++++
 include/uapi/linux/iommufd.h            | 18 +++++++++
 5 files changed, 82 insertions(+)
drivers/iommu/iommufd/iommufd_private.h +1 -0
index fce68714c80f..e08186f1d102 100644
@@ -697,6 +697,7 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj);
 void iommufd_vdevice_abort(struct iommufd_object *obj);
 int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
 void iommufd_hw_queue_destroy(struct iommufd_object *obj);
+int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd);
 
 static inline struct iommufd_vdevice *
 iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id)
drivers/iommu/iommufd/main.c +3 -0
index 89830da8b418..4f2a1995bd1f 100644
@@ -410,6 +410,7 @@ union ucmd_buffer {
 	struct iommu_veventq_alloc veventq;
 	struct iommu_vfio_ioas vfio_ioas;
 	struct iommu_viommu_alloc viommu;
+	struct iommu_vdevice_tsm_op tsm_op;
 #ifdef CONFIG_IOMMUFD_TEST
 	struct iommu_test_cmd test;
 #endif
@@ -471,6 +472,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
 		 __reserved),
 	IOCTL_OP(IOMMU_VIOMMU_ALLOC, iommufd_viommu_alloc_ioctl,
 		 struct iommu_viommu_alloc, out_viommu_id),
+	IOCTL_OP(IOMMU_VDEVICE_TSM_OP, iommufd_vdevice_tsm_op_ioctl,
+		 struct iommu_vdevice_tsm_op, vdevice_id),
 #ifdef CONFIG_IOMMUFD_TEST
 	IOCTL_OP(IOMMU_TEST_CMD, iommufd_test, struct iommu_test_cmd, last),
 #endif
drivers/iommu/iommufd/viommu.c +50 -0

59 lines not on this page · raw message

drivers/vfio/pci/vfio_pci_core.c +10 -0
index bee3cf3226e9..afdb39c6aefd 100644
@@ -694,6 +694,16 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
 #if IS_ENABLED(CONFIG_EEH)
 	eeh_dev_release(vdev->pdev);
 #endif
+
+#if 0
+	/*
+	 * destroy vdevice which involves tsm unbind before we disable pci disable
+	 * A MSE/BME clear will transition the device to error state.
+	 */
+	if (core_vdev->iommufd_device)
+		iommufd_device_tombstone_vdevice(core_vdev->iommufd_device);
+#endif
+
 	vfio_pci_core_disable(vdev);
 
 	mutex_lock(&vdev->igate);
include/uapi/linux/iommufd.h +18 -0
index 9014c61a97d4..8b1fbf1ef25c 100644
@@ -57,6 +57,7 @@ enum {
 	IOMMUFD_CMD_IOAS_CHANGE_PROCESS = 0x92,
 	IOMMUFD_CMD_VEVENTQ_ALLOC = 0x93,
 	IOMMUFD_CMD_HW_QUEUE_ALLOC = 0x94,
+	IOMMUFD_CMD_VDEVICE_TSM_OP = 0x95,
 };
 
 /**
@@ -1127,6 +1128,23 @@ enum iommu_veventq_flag {
 	IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0),
 };
 
+/**
+ * struct iommu_vdevice_tsm_OP - ioctl(IOMMU_VDEVICE_TSM_OP)
+ * @size: sizeof(struct iommu_vdevice_tsm_OP)
+ * @op: Either TSM_BIND or TSM_UNBIMD
+ * @flags: Must be 0
+ * @vdevice_id: Object handle for the vDevice. Returned from IOMMU_VDEVICE_ALLOC
+ */
+struct iommu_vdevice_tsm_op {
+	__u32 size;
+	__u32 op;
+	__u32 flags;
+	__u32 vdevice_id;
+};
+#define IOMMU_VDEVICE_TSM_OP	_IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_OP)
+#define IOMMU_VDEICE_TSM_BIND		0x1
+#define IOMMU_VDEICE_TSM_UNBIND		0x2
+
 /**
  * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status
  * @flags: Combination of enum iommu_veventq_flag
Signature · 1 line
2.43.0
author maintainer
#47 lore raw

RFC PATCH 9/38 iommufd/vdevice: Add TSM Guest request uAPI

In reply to Aneesh Kumar K.V (Arm)

Add TSM Guest request uAPI against iommufd_vdevice to forward various TSM attestation & acceptance requests from guest to TSM driver/secure firmware. This uAPI takes function only after TSM Bind.

After a vPCI device is locked down by TSM Bind, CoCo VM should attest and accept the device in its TEE. These operations needs interaction with secure firmware and the device, but doesn't impact the device management from host's POV. It doesn't change the fact that host should not touch some part of the device (see TDISP spec) to keep the trusted assignment, and host could exit trusted assignment and roll back everything by TSM Unbind.

So the TSM Guest request becomes a passthrough channel for CoCo VM to exchange request/response blobs with TSM driver/secure firmware. The definition of this IOCTL illustates this idea.

Based on changes from: Alexey Kardashevskiy <[email protected]>

---
 drivers/iommu/iommufd/iommufd_private.h |  1 +
 drivers/iommu/iommufd/main.c            |  3 ++
 drivers/iommu/iommufd/viommu.c          | 40 +++++++++++++++++++++++++
 drivers/pci/tsm.c                       | 15 ++++++++--
 drivers/virt/coco/tsm-core.c            |  9 ++++++
 include/linux/pci-tsm.h                 | 30 ++-----------------
 include/linux/tsm.h                     | 23 ++++++++++++++
 include/uapi/linux/iommufd.h            | 28 +++++++++++++++++
 8 files changed, 120 insertions(+), 29 deletions(-)
drivers/iommu/iommufd/iommufd_private.h +1 -0
index e08186f1d102..0c0d96135432 100644
@@ -698,6 +698,7 @@ void iommufd_vdevice_abort(struct iommufd_object *obj);
 int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
 void iommufd_hw_queue_destroy(struct iommufd_object *obj);
 int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd);
+int iommufd_vdevice_tsm_guest_request_ioctl(struct iommufd_ucmd *ucmd);
 
 static inline struct iommufd_vdevice *
 iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id)
drivers/iommu/iommufd/main.c +3 -0
index 4f2a1995bd1f..65e60da9caef 100644
@@ -411,6 +411,7 @@ union ucmd_buffer {
 	struct iommu_vfio_ioas vfio_ioas;
 	struct iommu_viommu_alloc viommu;
 	struct iommu_vdevice_tsm_op tsm_op;
+	struct iommu_vdevice_tsm_guest_request gr;
 #ifdef CONFIG_IOMMUFD_TEST
 	struct iommu_test_cmd test;
 #endif
@@ -474,6 +475,8 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
 		 struct iommu_viommu_alloc, out_viommu_id),
 	IOCTL_OP(IOMMU_VDEVICE_TSM_OP, iommufd_vdevice_tsm_op_ioctl,
 		 struct iommu_vdevice_tsm_op, vdevice_id),
+	IOCTL_OP(IOMMU_VDEVICE_TSM_GUEST_REQUEST, iommufd_vdevice_tsm_guest_request_ioctl,
+		 struct iommu_vdevice_tsm_guest_request, resp_uptr),
 #ifdef CONFIG_IOMMUFD_TEST
 	IOCTL_OP(IOMMU_TEST_CMD, iommufd_test, struct iommu_test_cmd, last),
 #endif
drivers/iommu/iommufd/viommu.c +40 -0
index c934312e5397..9f4d4d69b82b 100644
@@ -515,9 +515,49 @@ int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd)
 	iommufd_put_object(ucmd->ictx, &vdev->obj);
 	return rc;
 }
+
+int iommufd_vdevice_tsm_guest_request_ioctl(struct iommufd_ucmd *ucmd)
+{
+	struct iommu_vdevice_tsm_guest_request *cmd = ucmd->cmd;
+	struct tsm_guest_req_info info = {
+		.type = cmd->type,
+		.type_info = u64_to_user_ptr(cmd->type_info_uptr),
+		.type_info_len = cmd->type_info_len,
+		.req = u64_to_user_ptr(cmd->req_uptr),
+		.req_len = cmd->req_len,
+		.resp = u64_to_user_ptr(cmd->resp_uptr),
+		.resp_len = cmd->resp_len,
+	};
+	struct iommufd_vdevice *vdev;
+	int rc;
+
+	vdev = container_of(iommufd_get_object(ucmd->ictx, cmd->vdevice_id,
+					       IOMMUFD_OBJ_VDEVICE),
+			    struct iommufd_vdevice, obj);
+	if (IS_ERR(vdev))
+		return PTR_ERR(vdev);
+
+	rc = tsm_guest_req(vdev->idev->dev, &info);
+	if (rc)
+		goto err_out;
+
+	cmd->resp_len = info.resp_len;
+	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
+err_out:
+	iommufd_put_object(ucmd->ictx, &vdev->obj);
+	return rc;
+}
+
 #else /* !CONFIG_TSM */
+
 int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd)
 {
 	return -ENODEV;
 }
+
+int iommufd_vdevice_tsm_guest_request_ioctl(struct iommufd_ucmd *ucmd)
+{
+	return -ENODEV;
+}
+
 #endif
drivers/pci/tsm.c +13 -2
index 80607082b7f0..896ef0f5fbe7 100644
@@ -861,7 +861,7 @@ int pci_tsm_unbind(struct pci_dev *pdev)
 EXPORT_SYMBOL_GPL(pci_tsm_unbind);
 
 /**
- * pci_tsm_guest_req - VFIO/IOMMUFD helper to handle guest requests
+ * pci_tsm_guest_req - IOMMUFD helper to handle guest requests
  * @pdev: @pdev representing a bound tdi
  * @info: envelope for the request
  *
@@ -871,11 +871,12 @@ EXPORT_SYMBOL_GPL(pci_tsm_unbind);
  * posts to userspace (e.g. QEMU) that holds the host-to-guest RID
  * mapping.
  */
-int pci_tsm_guest_req(struct pci_dev *pdev, struct pci_tsm_guest_req_info *info)
+static int __pci_tsm_guest_req(struct pci_dev *pdev, struct tsm_guest_req_info *info)
 {
 	struct pci_tdi *tdi;
 	int rc;
 
+
 	lockdep_assert_held_read(&pci_tsm_rwsem);
 
 	if (!pdev->tsm)
@@ -899,4 +900,14 @@ int pci_tsm_guest_req(struct pci_dev *pdev, struct pci_tsm_guest_req_info *info)
 
 	return 0;
 }
+
+int pci_tsm_guest_req(struct pci_dev *pdev, struct tsm_guest_req_info *info)
+{
+	struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock();
+	if (!lock)
+		return -EINTR;
+
+	return __pci_tsm_guest_req(pdev, info);
+}
+
 EXPORT_SYMBOL_GPL(pci_tsm_guest_req);
drivers/virt/coco/tsm-core.c +9 -0
index 0a7c9aa46c56..32b1235518b4 100644
@@ -134,6 +134,15 @@ int tsm_unbind(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(tsm_unbind);
 
+int tsm_guest_req(struct device *dev, struct tsm_guest_req_info *info)
+{
+	if (!dev_is_pci(dev))
+		return -EINVAL;
+
+	return pci_tsm_guest_req(to_pci_dev(dev), info);
+}
+EXPORT_SYMBOL_GPL(tsm_guest_req);
+
 static void tsm_release(struct device *dev)
 {
 	struct tsm_core_dev *core = container_of(dev, typeof(*core), dev);
include/linux/pci-tsm.h +3 -27
index 7639e7963681..530f8b3093f8 100644
@@ -108,31 +108,7 @@ static inline bool is_pci_tsm_pf0(struct pci_dev *pdev)
 	return PCI_FUNC(pdev->devfn) == 0;
 }
 
-enum pci_tsm_guest_req_type {
-	PCI_TSM_GUEST_REQ_TDXC,
-};
-
-/**
- * struct pci_tsm_guest_req_info - parameter for pci_tsm_ops.guest_req()
- * @type: identify the format of the following blobs
- * @type_info: extra input/output info, e.g. firmware error code
- * @type_info_len: the size of @type_info
- * @req: request data buffer filled by guest
- * @req_len: the size of @req filled by guest
- * @resp: response data buffer filled by host
- * @resp_len: for input, the size of @resp buffer filled by guest
- *	      for output, the size of actual response data filled by host
- */
-struct pci_tsm_guest_req_info {
-	enum pci_tsm_guest_req_type type;
-	void *type_info;
-	size_t type_info_len;
-	void *req;
-	size_t req_len;
-	void *resp;
-	size_t resp_len;
-};
-
+struct tsm_guest_req_info;
 /**
  * struct pci_tsm_ops - Low-level TSM-exported interface to the PCI core
  * @probe: probe/accept device for tsm operation, setup DSM context
@@ -160,7 +136,7 @@ struct pci_tsm_ops {
 				struct kvm *kvm, u64 tdi_id);
 	void (*unbind)(struct pci_tdi *tdi);
 	int (*guest_req)(struct pci_dev *pdev,
-			 struct pci_tsm_guest_req_info *info);
+			 struct tsm_guest_req_info *info);
 	int (*accept)(struct pci_dev *pdev);
 };
 
@@ -180,7 +156,7 @@ int pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm);
 int pci_tsm_pf0_initialize(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm);
 int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id);
 int pci_tsm_unbind(struct pci_dev *pdev);
-int pci_tsm_guest_req(struct pci_dev *pdev, struct pci_tsm_guest_req_info *info);
+int pci_tsm_guest_req(struct pci_dev *pdev, struct tsm_guest_req_info *info);
 #else
 static inline int pci_tsm_core_register(const struct pci_tsm_ops *ops,
 					const struct attribute_group *grp)
include/linux/tsm.h +23 -0
index 0aab8d037e71..497a3b4df5a0 100644
@@ -123,4 +123,27 @@ int tsm_ide_stream_register(struct pci_dev *pdev, struct pci_ide *ide);
 void tsm_ide_stream_unregister(struct pci_ide *ide);
 int tsm_bind(struct device *dev, struct kvm *kvm, u64 tdi_id);
 int tsm_unbind(struct device *dev);
+
+/**
+ * struct tsm_guest_req_info - parameter for pci_tsm_ops.guest_req()
+ * @type: identify the format of the following blobs
+ * @type_info: extra input/output info, e.g. firmware error code
+ * @type_info_len: the size of @type_info
+ * @req: request data buffer filled by guest
+ * @req_len: the size of @req filled by guest
+ * @resp: response data buffer filled by host
+ * @resp_len: for input, the size of @resp buffer filled by guest
+ *	      for output, the size of actual response data filled by host
+ */
+struct tsm_guest_req_info {
+	u32 type;
+	void __user *type_info;
+	size_t type_info_len;
+	void __user *req;
+	size_t req_len;
+	void __user *resp;
+	size_t resp_len;
+};
+
+int tsm_guest_req(struct device *dev, struct tsm_guest_req_info *info);
 #endif /* __TSM_H */
include/uapi/linux/iommufd.h +28 -0
index 8b1fbf1ef25c..56542cfcfa38 100644
@@ -58,6 +58,7 @@ enum {
 	IOMMUFD_CMD_VEVENTQ_ALLOC = 0x93,
 	IOMMUFD_CMD_HW_QUEUE_ALLOC = 0x94,
 	IOMMUFD_CMD_VDEVICE_TSM_OP = 0x95,
+	IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST = 0x96,
 };
 
 /**
@@ -1320,4 +1321,31 @@ struct iommu_hw_queue_alloc {
 	__aligned_u64 length;
 };
 #define IOMMU_HW_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_HW_QUEUE_ALLOC)
+
+/**
+ * struct iommu_vdevice_tsm_guest_request - ioctl(IOMMU_VDEVICE_TSM_GUEST_REQUEST)
+ * @size: sizeof(struct iommu_vdevice_tsm_guest_request)
+ * @vdevice_id: vDevice ID the guest request is for
+ * @type: identify the format of the following blobs
+ * @type_info_len: the blob size for @type_info_uptr
+ * @req_len: the blob size for @req_uptr, filled by guest
+ * @resp_len: for input, the blob size for @resp_uptr, filled by guest
+ *	      for output, the size of actual response data, filled by host
+ * @type_info_uptr: extra input/output info, e.g. firmware error code
+ * @req_uptr: request data buffer filled by guest
+ * @resp_uptr: response data buffer filled by host
+ */
+struct iommu_vdevice_tsm_guest_request {
+	__u32 size;
+	__u32 vdevice_id;
+	__u32 type;
+	__u32 type_info_len;
+	__u32 req_len;
+	__u32 resp_len;
+	__aligned_u64 type_info_uptr;
+	__aligned_u64 req_uptr;
+	__aligned_u64 resp_uptr;
+};
+#define IOMMU_VDEVICE_TSM_GUEST_REQUEST _IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST)
+
 #endif
Signature · 1 line
2.43.0
author maintainer
#49 lore raw

RFC PATCH 10/38 iommufd/vdevice: Add TSM map ioctl

In reply to Aneesh Kumar K.V (Arm)

With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent

---
 arch/arm64/kvm/mmu.c                    | 45 +++++++++++++++++++++++++
 drivers/iommu/iommufd/iommufd_private.h |  1 +
 drivers/iommu/iommufd/main.c            |  4 +++
 drivers/iommu/iommufd/viommu.c          | 43 +++++++++++++++++++++++
 include/linux/kvm_host.h                |  1 +
 include/uapi/linux/iommufd.h            | 10 ++++++
 6 files changed, 104 insertions(+)
arch/arm64/kvm/mmu.c +45 -0
index c866891fd8f9..8788d24095d6 100644
@@ -1530,6 +1530,51 @@ static int realm_map_ipa(struct kvm *kvm, phys_addr_t ipa,
 	return realm_map_protected(realm, ipa, pfn, map_size, memcache);
 }
 
+int kvm_map_private_memory(struct kvm *kvm, phys_addr_t start_gpa,
+			   phys_addr_t end_gpa)
+{
+	struct kvm_mmu_memory_cache cache = { .gfp_zero = __GFP_ZERO };
+	struct kvm_s2_mmu *mmu = &kvm->arch.mmu;
+	struct kvm_memory_slot *memslot;
+	phys_addr_t addr;
+	struct page *page;
+	kvm_pfn_t pfn;
+	int ret = 0, idx;
+	gfn_t gfn;
+
+	idx = srcu_read_lock(&kvm->srcu);
+	for (addr = start_gpa; addr < end_gpa; addr += PAGE_SIZE) {
+
+		ret = kvm_mmu_topup_memory_cache(&cache,
+						 kvm_mmu_cache_min_pages(mmu));
+		if (ret)
+			break;
+
+		gfn = addr >> PAGE_SHIFT;
+
+		memslot = gfn_to_memslot(kvm, gfn);
+		if (!kvm_slot_can_be_private(memslot)) {
+			ret = -EINVAL;
+			break;
+		}
+		/* should we check if kvm_mem_is_private()? */
+		ret = kvm_gmem_get_pfn(kvm, memslot, gfn, &pfn, &page, NULL);
+		if (ret)
+			break;
+
+		/* should we hold kvm_fault_lock()? */
+		ret = realm_map_ipa(kvm, addr, pfn, PAGE_SIZE, KVM_PGTABLE_PROT_W,
+				    &cache);
+		if (ret) {
+			put_page(page);
+			break;
+		}
+	}
+	kvm_mmu_free_memory_cache(&cache);
+	srcu_read_unlock(&kvm->srcu, idx);
+	return ret;
+}
+
 static int private_memslot_fault(struct kvm_vcpu *vcpu,
 				 phys_addr_t fault_ipa,
 				 struct kvm_memory_slot *memslot)
drivers/iommu/iommufd/iommufd_private.h +1 -0
index 0c0d96135432..34f3ae0e0cd1 100644
@@ -698,6 +698,7 @@ void iommufd_vdevice_abort(struct iommufd_object *obj);
 int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
 void iommufd_hw_queue_destroy(struct iommufd_object *obj);
 int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd);
+int iommufd_vdevice_tsm_map_ioctl(struct iommufd_ucmd *ucmd);
 int iommufd_vdevice_tsm_guest_request_ioctl(struct iommufd_ucmd *ucmd);
 
 static inline struct iommufd_vdevice *
drivers/iommu/iommufd/main.c +4 -0
index 65e60da9caef..388d11334994 100644
@@ -411,6 +411,7 @@ union ucmd_buffer {
 	struct iommu_vfio_ioas vfio_ioas;
 	struct iommu_viommu_alloc viommu;
 	struct iommu_vdevice_tsm_op tsm_op;
+	struct iommu_vdevice_tsm_map tsm_map;
 	struct iommu_vdevice_tsm_guest_request gr;
 #ifdef CONFIG_IOMMUFD_TEST
 	struct iommu_test_cmd test;
@@ -475,6 +476,9 @@ static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
 		 struct iommu_viommu_alloc, out_viommu_id),
 	IOCTL_OP(IOMMU_VDEVICE_TSM_OP, iommufd_vdevice_tsm_op_ioctl,
 		 struct iommu_vdevice_tsm_op, vdevice_id),
+	IOCTL_OP(IOMMU_VDEVICE_TSM_MAP, iommufd_vdevice_tsm_map_ioctl,
+		 struct iommu_vdevice_tsm_map, vdevice_id),
+
 	IOCTL_OP(IOMMU_VDEVICE_TSM_GUEST_REQUEST, iommufd_vdevice_tsm_guest_request_ioctl,
 		 struct iommu_vdevice_tsm_guest_request, resp_uptr),
 #ifdef CONFIG_IOMMUFD_TEST
drivers/iommu/iommufd/viommu.c +43 -0

55 lines not on this page · raw message

include/linux/kvm_host.h +1 -0
index 3bde4fb5c6aa..bfdfb4f32d28 100644
@@ -2602,4 +2602,5 @@ static inline int kvm_enable_virtualization(void) { return 0; }
 static inline void kvm_disable_virtualization(void) { }
 #endif
 
+int kvm_map_private_memory(struct kvm *kvm, phys_addr_t start_gpa, phys_addr_t end_gpa);
 #endif
include/uapi/linux/iommufd.h +10 -0
index 56542cfcfa38..75056d1f141d 100644
@@ -59,6 +59,7 @@ enum {
 	IOMMUFD_CMD_HW_QUEUE_ALLOC = 0x94,
 	IOMMUFD_CMD_VDEVICE_TSM_OP = 0x95,
 	IOMMUFD_CMD_VDEVICE_TSM_GUEST_REQUEST = 0x96,
+	IOMMUFD_CMD_VDEVICE_TSM_MAP = 0x97,
 };
 
 /**
@@ -1146,6 +1147,15 @@ struct iommu_vdevice_tsm_op {
 #define IOMMU_VDEICE_TSM_BIND		0x1
 #define IOMMU_VDEICE_TSM_UNBIND		0x2
 
+struct iommu_vdevice_tsm_map {
+	__u32 size;
+	__u32 flags;
+	__u64 start_gpa;
+	__u64 end_gpa;
+	__u32 vdevice_id;
+};
+#define IOMMU_VDEVICE_TSM_MAP	_IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_MAP)
+
 /**
  * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status
  * @flags: Combination of enum iommu_veventq_flag
Signature · 1 line
2.43.0
author maintainer
#57 lore raw

RFC PATCH 11/38 KVM: arm64: CCA: register host tsm platform device

In reply to Aneesh Kumar K.V (Arm)

Register a platform device if the CCA DA feature is supported. A driver for this platform device will further drive the CCA DA workflow.

---
 arch/arm64/include/asm/rmi_smc.h |  3 +++
 arch/arm64/kvm/rme.c             | 16 ++++++++++++++++
 2 files changed, 19 insertions(+)
arch/arm64/include/asm/rmi_smc.h +3 -0
index 504009a42035..42708d500048 100644
@@ -12,6 +12,8 @@
 
 #include <linux/arm-smccc.h>
 
+#define RMI_DEV_NAME "arm-rmi-dev"
+
 #define SMC_RMI_CALL(func)				\
 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,		\
 			   ARM_SMCCC_SMC_64,		\
@@ -87,6 +89,7 @@ enum rmi_ripas {
 #define RMI_FEATURE_REGISTER_0_GICV3_NUM_LRS	GENMASK(37, 34)
 #define RMI_FEATURE_REGISTER_0_MAX_RECS_ORDER	GENMASK(41, 38)
 #define RMI_FEATURE_REGISTER_0_Reserved		GENMASK(63, 42)
+#define RMI_FEATURE_REGISTER_0_DA		BIT(42)
 
 #define RMI_REALM_PARAM_FLAG_LPA2		BIT(0)
 #define RMI_REALM_PARAM_FLAG_SVE		BIT(1)
arch/arm64/kvm/rme.c +16 -0
index ec8093ec2da3..d1c147aba2ed 100644
@@ -4,6 +4,7 @@
  */
 
 #include <linux/kvm_host.h>
+#include <linux/platform_device.h>
 
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_mmu.h>
@@ -1724,6 +1725,18 @@ int kvm_init_realm_vm(struct kvm *kvm)
 	return 0;
 }
 
+static struct platform_device cca_host_dev = {
+	.name = RMI_DEV_NAME,
+	.id = PLATFORM_DEVID_NONE
+};
+
+static void rmm_tsm_init(void)
+{
+	if (!platform_device_register(&cca_host_dev))
+		pr_info("CCA host DA platform device initialized.\n");
+
+}
+
 void kvm_init_rme(void)
 {
 	if (PAGE_SIZE != SZ_4K)
@@ -1737,6 +1750,9 @@ void kvm_init_rme(void)
 	if (WARN_ON(rmi_features(0, &rmm_feat_reg0)))
 		return;
 
+	if (rme_has_feature(RMI_FEATURE_REGISTER_0_DA))
+		rmm_tsm_init();
+
 	if (rme_vmid_init())
 		return;
 
Signature · 1 line
2.43.0
author maintainer
#90 lore raw

RFC PATCH 12/38 coco: host: arm64: CCA host platform device driver

In reply to Aneesh Kumar K.V (Arm)

This driver registers the pci_tsm_ops with tsm subsystem.

---
 drivers/virt/coco/Kconfig                |   2 +
 drivers/virt/coco/Makefile               |   1 +
 drivers/virt/coco/arm-cca-host/Kconfig   |  12 ++
 drivers/virt/coco/arm-cca-host/Makefile  |   5 +
 drivers/virt/coco/arm-cca-host/arm-cca.c | 209 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  29 ++++
 6 files changed, 258 insertions(+)
 create mode 100644 drivers/virt/coco/arm-cca-host/Kconfig
 create mode 100644 drivers/virt/coco/arm-cca-host/Makefile
 create mode 100644 drivers/virt/coco/arm-cca-host/arm-cca.c
 create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.h
drivers/virt/coco/Kconfig +2 -0
index 57248b088545..43e9508301bf 100644
@@ -15,5 +15,7 @@ source "drivers/virt/coco/arm-cca-guest/Kconfig"
 
 source "drivers/virt/coco/guest/Kconfig"
 
+source "drivers/virt/coco/arm-cca-host/Kconfig"
+
 config TSM
 	tristate
drivers/virt/coco/Makefile +1 -0
index 04e124b2d7cf..d0a859dd9eaf 100644
@@ -11,3 +11,4 @@ obj-$(CONFIG_ARM_CCA_GUEST)	+= arm-cca-guest/
 
 obj-$(CONFIG_TSM) 		+= tsm-core.o
 obj-y				+= guest/
+obj-$(CONFIG_ARM_CCA_HOST)	+= arm-cca-host/
drivers/virt/coco/arm-cca-host/Kconfig new file +12 -0
new file mode 100644
index 000000000000..0f19fbf47613
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# TSM (TEE Security Manager) host drivers
+#
+config ARM_CCA_HOST
+	tristate "Arm CCA Host driver"
+	depends on ARM64
+	depends on PCI_TSM
+	select TSM
+
+	help
+	  The driver provides TSM backend for ARM CCA
drivers/virt/coco/arm-cca-host/Makefile new file +5 -0
new file mode 100644
index 000000000000..ad353b07e95a
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+obj-$(CONFIG_ARM_CCA_HOST) += arm-cca-host.o
+
+arm-cca-host-$(CONFIG_TSM) +=  arm-cca.o
drivers/virt/coco/arm-cca-host/arm-cca.c new file +209 -0

209 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h new file +29 -0
new file mode 100644
index 000000000000..840cb584acdd
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 ARM Ltd.
+ */
+
+#ifndef RMM_DA_H_
+#define RMM_DA_H_
+
+#include <linux/pci.h>
+#include <linux/pci-ide.h>
+#include <linux/pci-tsm.h>
+#include <asm/rmi_smc.h>
+
+struct cca_host_dsc_pf0 {
+	struct pci_tsm_pf0 pci;
+	struct pci_ide *sel_stream;
+};
+
+static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
+{
+	struct pci_tsm *tsm = pdev->tsm;
+
+	if (!tsm || pdev->is_virtfn || !is_pci_tsm_pf0(pdev))
+		return NULL;
+
+	return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm);
+}
+
+#endif
Signature · 1 line
2.43.0
author maintainer
#97 lore raw

RFC PATCH 13/38 coco: host: arm64: Create a PDEV with rmm

In reply to Aneesh Kumar K.V (Arm)

Create the realm physical device with RMM.

---
 arch/arm64/include/asm/rmi_cmds.h        |  31 +++++
 arch/arm64/include/asm/rmi_smc.h         |  72 ++++++++++-
 drivers/virt/coco/arm-cca-host/Makefile  |   2 +-
 drivers/virt/coco/arm-cca-host/arm-cca.c |  10 +-
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 150 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |   5 +
 6 files changed, 267 insertions(+), 3 deletions(-)
 create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.c
arch/arm64/include/asm/rmi_cmds.h +31 -0
index ef53147c1984..f0817bd3bab4 100644
@@ -505,4 +505,35 @@ static inline int rmi_rtt_unmap_unprotected(unsigned long rd,
 	return res.a0;
 }
 
+static inline unsigned long rmi_pdev_aux_count(unsigned long flags, u64 *aux_count)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_AUX_COUNT, flags, &res);
+
+	*aux_count = res.a1;
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_create(unsigned long pdev_phys,
+					    unsigned long pdev_params_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_CREATE,
+			     pdev_phys, pdev_params_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_get_state(unsigned long pdev_phys, unsigned long *state)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_GET_STATE, pdev_phys, &res);
+
+	*state = res.a1;
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +71 -1

88 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/Makefile +1 -1
index ad353b07e95a..8409220a6510 100644
@@ -2,4 +2,4 @@
 #
 obj-$(CONFIG_ARM_CCA_HOST) += arm-cca-host.o
 
-arm-cca-host-$(CONFIG_TSM) +=  arm-cca.o
+arm-cca-host-$(CONFIG_TSM) +=  arm-cca.o rmm-da.o
drivers/virt/coco/arm-cca-host/arm-cca.c +9 -1
index c8b0e6db1f47..84d97dd41191 100644
@@ -124,7 +124,15 @@ static int cca_tsm_connect(struct pci_dev *pdev)
 	rc = tsm_ide_stream_register(pdev, ide);
 	if (rc)
 		goto err_tsm;
-
+	/*
+	 * Take a module reference so that we won't call unregister
+	 * without rme_unasign_device
+	 */
+	if (!try_module_get(THIS_MODULE)) {
+		rc = -ENXIO;
+		goto err_tsm;
+	}
+	rme_asign_device(pdev);
 	/*
 	 * Once ide is setup enable the stream at endpoint
 	 * Root port will be done by RMM
drivers/virt/coco/arm-cca-host/rmm-da.c new file +150 -0

150 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h +5 -0
index 840cb584acdd..179ba68f2430 100644
@@ -14,6 +14,10 @@
 struct cca_host_dsc_pf0 {
 	struct pci_tsm_pf0 pci;
 	struct pci_ide *sel_stream;
+
+	void *rmm_pdev;
+	int num_aux;
+	void *aux[MAX_PDEV_AUX_GRANULES];
 };
 
 static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
@@ -26,4 +30,5 @@ static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
 	return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm);
 }
 
+int rme_asign_device(struct pci_dev *pdev);
 #endif
Signature · 1 line
2.43.0
author maintainer
#103 lore raw

RFC PATCH 14/38 coco: host: arm64: Device communication support

In reply to Aneesh Kumar K.V (Arm)

Add helpers for device communication from RMM

---
 arch/arm64/include/asm/rmi_cmds.h        |  11 ++
 arch/arm64/include/asm/rmi_smc.h         |  49 ++++++
 drivers/virt/coco/arm-cca-host/arm-cca.c |  45 ++++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 198 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  41 +++++
 5 files changed, 344 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +11 -0
index f0817bd3bab4..eb0034a675bb 100644
@@ -536,4 +536,15 @@ static inline unsigned long rmi_pdev_get_state(unsigned long pdev_phys, unsigned
 	return res.a0;
 }
 
+static inline unsigned long rmi_pdev_communicate(unsigned long pdev_phys,
+						 unsigned long pdev_comm_data_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_COMMUNICATE,
+			     pdev_phys, pdev_comm_data_phys, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +49 -0

59 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/arm-cca.c +45 -0

63 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.c +198 -0

201 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h +41 -0

61 lines not on this page · raw message

Signature · 1 line
2.43.0
author maintainer
#108 lore raw

RFC PATCH 15/38 coco: host: arm64: Stop and destroy the physical device

In reply to Aneesh Kumar K.V (Arm)

Add support for stopping and destroying physical devices.

---
 arch/arm64/include/asm/rmi_cmds.h        | 18 ++++++++++++++++++
 arch/arm64/include/asm/rmi_smc.h         |  2 ++
 drivers/virt/coco/arm-cca-host/arm-cca.c |  3 +++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 21 +++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  1 +
 5 files changed, 45 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +18 -0
index eb0034a675bb..d4ea9f8363f5 100644
@@ -547,4 +547,22 @@ static inline unsigned long rmi_pdev_communicate(unsigned long pdev_phys,
 	return res.a0;
 }
 
+static inline unsigned long rmi_pdev_stop(unsigned long pdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_STOP, pdev_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_destroy(unsigned long pdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_DESTROY, pdev_phys, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +2 -0
index 8bece465b670..9f25a876238e 100644
@@ -49,7 +49,9 @@
 
 #define SMC_RMI_PDEV_COMMUNICATE        SMC_RMI_CALL(0x0175)
 #define SMC_RMI_PDEV_CREATE             SMC_RMI_CALL(0x0176)
+#define SMC_RMI_PDEV_DESTROY		SMC_RMI_CALL(0x0177)
 #define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
+#define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
 
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0
drivers/virt/coco/arm-cca-host/arm-cca.c +3 -0
index 294a6ef60d5f..c65b81f0706f 100644
@@ -210,12 +210,15 @@ static void cca_tsm_disconnect(struct pci_dev *pdev)
 	ide = dsc_pf0->sel_stream;
 	dsc_pf0->sel_stream = NULL;
 	pci_ide_stream_disable(pdev, ide);
+	rme_unassign_device(pdev);
+	module_put(THIS_MODULE);
 	tsm_ide_stream_unregister(ide);
 	pci_ide_stream_teardown(rp, ide);
 	pci_ide_stream_teardown(pdev, ide);
 	pci_ide_stream_unregister(ide);
 	clear_bit(ide->stream_id, cca_stream_ids);
 	pci_ide_stream_free(ide);
+	free_dev_communication_buffers(&dsc_pf0->comm_data);
 }
 
 static const struct pci_tsm_ops cca_pci_ops = {
drivers/virt/coco/arm-cca-host/rmm-da.c +21 -0
index d123940ce82e..ec8c5bfcee35 100644
@@ -346,3 +346,24 @@ int schedule_rme_ide_setup(struct pci_dev *pdev)
 
 	return 0;
 }
+
+void rme_unassign_device(struct pci_dev *pdev)
+{
+	unsigned long ret;
+	unsigned long state;
+	phys_addr_t rmm_pdev_phys;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+
+	dsc_pf0 = to_cca_dsc_pf0(pdev);
+	rmm_pdev_phys = virt_to_phys(dsc_pf0->rmm_pdev);
+	ret = rmi_pdev_stop(rmm_pdev_phys);
+	if (WARN_ON(ret != RMI_SUCCESS))
+		return;
+
+	state = do_pdev_communicate(pdev->tsm, RMI_PDEV_STOPPED);
+	/* ignore the error state and destroy the device */
+	WARN_ON(state != RMI_PDEV_STOPPED);
+	ret = rmi_pdev_destroy(rmm_pdev_phys);
+	if (WARN_ON(ret != RMI_SUCCESS))
+		return;
+}
drivers/virt/coco/arm-cca-host/rmm-da.h +1 -0
index b9ddc4d9112b..c401be55d770 100644
@@ -71,5 +71,6 @@ static inline struct cca_host_comm_data *to_cca_comm_data(struct pci_dev *pdev)
 }
 
 int rme_asign_device(struct pci_dev *pdev);
+void rme_unassign_device(struct pci_dev *pdev);
 int schedule_rme_ide_setup(struct pci_dev *pdev);
 #endif
Signature · 1 line
2.43.0
author maintainer
#111 lore raw

RFC PATCH 16/38 X.509: Make certificate parser public

In reply to Aneesh Kumar K.V (Arm)

From: Lukas Wunner <[email protected]>

The upcoming support for PCI device authentication with CMA-SPDM (PCIe r6.1 sec 6.31) requires validating the Subject Alternative Name in X.509 certificates.

High-level functions for X.509 parsing such as key_create_or_update() throw away the internal, low-level struct x509_certificate after extracting the struct public_key and public_key_signature from it. The Subject Alternative Name is thus inaccessible when using those functions.

Afford CMA-SPDM access to the Subject Alternative Name by making struct x509_certificate public, together with the functions for parsing an X.509 certificate into such a struct and freeing such a struct.

The private header file x509_parser.h previously included <linux/time.h> for the definition of time64_t. That definition was since moved to <linux/time64.h> by commit 361a3bf00582 ("time64: Add time64.h header and define struct timespec64"), so adjust the #include directive as part of the move to the new public header file <keys/x509-parser.h>.

No functional change intended.

---
 crypto/asymmetric_keys/x509_parser.h | 40 +--------------------
 include/keys/x509-parser.h           | 53 ++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 39 deletions(-)
 create mode 100644 include/keys/x509-parser.h
crypto/asymmetric_keys/x509_parser.h +1 -39
index 0688c222806b..39f1521b773d 100644
@@ -5,49 +5,11 @@
  * Written by David Howells ([email protected])
  */
 
-#include <linux/cleanup.h>
-#include <linux/time.h>
-#include <crypto/public_key.h>
-#include <keys/asymmetric-type.h>
-
-struct x509_certificate {
-	struct x509_certificate *next;
-	struct x509_certificate *signer;	/* Certificate that signed this one */
-	struct public_key *pub;			/* Public key details */
-	struct public_key_signature *sig;	/* Signature parameters */
-	char		*issuer;		/* Name of certificate issuer */
-	char		*subject;		/* Name of certificate subject */
-	struct asymmetric_key_id *id;		/* Issuer + Serial number */
-	struct asymmetric_key_id *skid;		/* Subject + subjectKeyId (optional) */
-	time64_t	valid_from;
-	time64_t	valid_to;
-	const void	*tbs;			/* Signed data */
-	unsigned	tbs_size;		/* Size of signed data */
-	unsigned	raw_sig_size;		/* Size of signature */
-	const void	*raw_sig;		/* Signature data */
-	const void	*raw_serial;		/* Raw serial number in ASN.1 */
-	unsigned	raw_serial_size;
-	unsigned	raw_issuer_size;
-	const void	*raw_issuer;		/* Raw issuer name in ASN.1 */
-	const void	*raw_subject;		/* Raw subject name in ASN.1 */
-	unsigned	raw_subject_size;
-	unsigned	raw_skid_size;
-	const void	*raw_skid;		/* Raw subjectKeyId in ASN.1 */
-	unsigned	index;
-	bool		seen;			/* Infinite recursion prevention */
-	bool		verified;
-	bool		self_signed;		/* T if self-signed (check unsupported_sig too) */
-	bool		unsupported_sig;	/* T if signature uses unsupported crypto */
-	bool		blacklisted;
-};
+#include <keys/x509-parser.h>
 
 /*
  * x509_cert_parser.c
  */
-extern void x509_free_certificate(struct x509_certificate *cert);
-DEFINE_FREE(x509_free_certificate, struct x509_certificate *,
-	    if (!IS_ERR(_T)) x509_free_certificate(_T))
-extern struct x509_certificate *x509_cert_parse(const void *data, size_t datalen);
 extern int x509_decode_time(time64_t *_t,  size_t hdrlen,
 			    unsigned char tag,
 			    const unsigned char *value, size_t vlen);
include/keys/x509-parser.h new file +53 -0

53 lines not on this page · raw message

Signature · 1 line
2.43.0
author maintainer
#112 lore raw

RFC PATCH 17/38 X.509: Parse Subject Alternative Name in certificates

In reply to Aneesh Kumar K.V (Arm)

From: Lukas Wunner <[email protected]>

The upcoming support for PCI device authentication with CMA-SPDM (PCIe r6.1 sec 6.31) requires validating the Subject Alternative Name in X.509 certificates.

Store a pointer to the Subject Alternative Name upon parsing for consumption by CMA-SPDM.

---
 crypto/asymmetric_keys/x509_cert_parser.c | 9 +++++++++
 include/keys/x509-parser.h                | 2 ++
 2 files changed, 11 insertions(+)
crypto/asymmetric_keys/x509_cert_parser.c +9 -0
index 2ffe4ae90bea..ac8a01c2b9fc 100644
@@ -571,6 +571,15 @@ int x509_process_extension(void *context, size_t hdrlen,
 		return 0;
 	}
 
+	if (ctx->last_oid == OID_subjectAltName) {
+		if (ctx->cert->raw_san)
+			return -EBADMSG;
+
+		ctx->cert->raw_san = v;
+		ctx->cert->raw_san_size = vlen;
+		return 0;
+	}
+
 	if (ctx->last_oid == OID_keyUsage) {
 		/*
 		 * Get hold of the keyUsage bit string
include/keys/x509-parser.h +2 -0
index 37436a5c7526..8e450befe3b9 100644
@@ -36,6 +36,8 @@ struct x509_certificate {
 	unsigned	raw_subject_size;
 	unsigned	raw_skid_size;
 	const void	*raw_skid;		/* Raw subjectKeyId in ASN.1 */
+	const void	*raw_san;		/* Raw subjectAltName in ASN.1 */
+	unsigned	raw_san_size;
 	unsigned	index;
 	bool		seen;			/* Infinite recursion prevention */
 	bool		verified;
Signature · 1 line
2.43.0
author maintainer
#113 lore raw

RFC PATCH 18/38 X.509: Move certificate length retrieval into new helper

In reply to Aneesh Kumar K.V (Arm)

From: Lukas Wunner <[email protected]>

The upcoming in-kernel SPDM library (Security Protocol and Data Model,
https://www.dmtf.org/dsp/DSP0274) needs to retrieve the length from ASN.1 DER-encoded X.509 certificates.

Such code already exists in x509_load_certificate_list(), so move it into a new helper for reuse by SPDM.

Export the helper so that SPDM can be tristate. (Some upcoming users of the SPDM libray may be modular, such as SCSI and ATA.)

No functional change intended.

---
 crypto/asymmetric_keys/x509_loader.c | 38 +++++++++++++++++++---------
 include/keys/asymmetric-type.h       |  2 ++
 2 files changed, 28 insertions(+), 12 deletions(-)
crypto/asymmetric_keys/x509_loader.c +26 -12
index a41741326998..25ff027fad1d 100644
@@ -4,28 +4,42 @@
 #include <linux/key.h>
 #include <keys/asymmetric-type.h>
 
+ssize_t x509_get_certificate_length(const u8 *p, unsigned long buflen)
+{
+	ssize_t plen;
+
+	/* Each cert begins with an ASN.1 SEQUENCE tag and must be more
+	 * than 256 bytes in size.
+	 */
+	if (buflen < 4)
+		return -EINVAL;
+
+	if (p[0] != 0x30 &&
+	    p[1] != 0x82)
+		return -EINVAL;
+
+	plen = (p[2] << 8) | p[3];
+	plen += 4;
+	if (plen > buflen)
+		return -EINVAL;
+
+	return plen;
+}
+EXPORT_SYMBOL_GPL(x509_get_certificate_length);
+
 int x509_load_certificate_list(const u8 cert_list[],
 			       const unsigned long list_size,
 			       const struct key *keyring)
 {
 	key_ref_t key;
 	const u8 *p, *end;
-	size_t plen;
+	ssize_t plen;
 
 	p = cert_list;
 	end = p + list_size;
 	while (p < end) {
-		/* Each cert begins with an ASN.1 SEQUENCE tag and must be more
-		 * than 256 bytes in size.
-		 */
-		if (end - p < 4)
-			goto dodgy_cert;
-		if (p[0] != 0x30 &&
-		    p[1] != 0x82)
-			goto dodgy_cert;
-		plen = (p[2] << 8) | p[3];
-		plen += 4;
-		if (plen > end - p)
+		plen = x509_get_certificate_length(p, end - p);
+		if (plen < 0)
 			goto dodgy_cert;
 
 		key = key_create_or_update(make_key_ref(keyring, 1),
include/keys/asymmetric-type.h +2 -0
index 69a13e1e5b2e..e2af07fec3c6 100644
@@ -84,6 +84,8 @@ extern struct key *find_asymmetric_key(struct key *keyring,
 				       const struct asymmetric_key_id *id_2,
 				       bool partial);
 
+ssize_t x509_get_certificate_length(const u8 *p, unsigned long buflen);
+
 int x509_load_certificate_list(const u8 cert_list[], const unsigned long list_size,
 			       const struct key *keyring);
 
Signature · 1 line
2.43.0
author maintainer
#115 lore raw

RFC PATCH 19/38 coco: host: arm64: set_pubkey support

In reply to Aneesh Kumar K.V (Arm)

Add changes to share the device's public key with the RMM.

---
 arch/arm64/include/asm/rmi_cmds.h       |   9 ++
 arch/arm64/include/asm/rmi_smc.h        |  18 +++
 drivers/virt/coco/arm-cca-host/Kconfig  |   4 +
 drivers/virt/coco/arm-cca-host/rmm-da.c | 150 ++++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h |   1 +
 5 files changed, 182 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +9 -0
index d4ea9f8363f5..aef0b0ee062e 100644
@@ -565,4 +565,13 @@ static inline unsigned long rmi_pdev_destroy(unsigned long pdev_phys)
 	return res.a0;
 }
 
+static inline unsigned long rmi_pdev_set_pubkey(unsigned long pdev_phys, unsigned long key_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_SET_PUBKEY, pdev_phys, key_phys, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +18 -0
index 9f25a876238e..4a5ba98c1c0d 100644
@@ -51,6 +51,7 @@
 #define SMC_RMI_PDEV_CREATE             SMC_RMI_CALL(0x0176)
 #define SMC_RMI_PDEV_DESTROY		SMC_RMI_CALL(0x0177)
 #define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
+#define SMC_RMI_PDEV_SET_PUBKEY		SMC_RMI_CALL(0x017b)
 #define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
 
 #define RMI_ABI_MAJOR_VERSION	1
@@ -389,4 +390,21 @@ struct rmi_dev_comm_data {
 	};
 };
 
+#define RMI_SIG_RSASSA_3072	0
+#define RMI_SIG_ECDSA_P256	1
+#define RMI_SIG_ECDSA_P384	2
+
+struct rmi_public_key_params {
+	union {
+		struct {
+			u8 public_key[1024];
+			u8 metadata[1024];
+			u64 public_key_len;
+			u64 metadata_len;
+			u8 rmi_signature_algorithm;
+		} __packed;
+		u8 padding[0x1000];
+	};
+};
+
 #endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/Kconfig +4 -0
index 0f19fbf47613..a5b777f0d50e 100644
@@ -6,7 +6,11 @@ config ARM_CCA_HOST
 	tristate "Arm CCA Host driver"
 	depends on ARM64
 	depends on PCI_TSM
+	select KEYS
+	select X509_CERTIFICATE_PARSER
 	select TSM
+	select CRYPTO_ECDSA
+	select CRYPTO_RSA
 
 	help
 	  The driver provides TSM backend for ARM CCA
drivers/virt/coco/arm-cca-host/rmm-da.c +150 -0

174 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h +1 -0
index c401be55d770..03c3149b8a98 100644
@@ -33,6 +33,7 @@ struct cca_host_dsc_pf0 {
 	int num_aux;
 	void *aux[MAX_PDEV_AUX_GRANULES];
 
+	uint8_t rmi_signature_algorithm;
 	struct {
 		struct cache_object cache;
 
Signature · 1 line
2.43.0
author maintainer
#119 lore raw

RFC PATCH 20/38 coco: host: arm64: Add support for creating a virtual device

In reply to Aneesh Kumar K.V (Arm)

Changes to support the creation of virtual device objects with RMM.

---
 arch/arm64/include/asm/rmi_cmds.h        | 13 +++++
 arch/arm64/include/asm/rmi_smc.h         | 30 +++++++++++
 drivers/virt/coco/arm-cca-host/Kconfig   |  1 +
 drivers/virt/coco/arm-cca-host/arm-cca.c | 30 +++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 67 ++++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  | 17 ++++++
 6 files changed, 158 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +13 -0
index aef0b0ee062e..7d91f847069b 100644
@@ -574,4 +574,17 @@ static inline unsigned long rmi_pdev_set_pubkey(unsigned long pdev_phys, unsigne
 	return res.a0;
 }
 
+static inline unsigned long rmi_vdev_create(unsigned long rd,
+					    unsigned long pdev_phys,
+					    unsigned long vdev_phys,
+					    unsigned long vdev_params_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_CREATE, rd, pdev_phys,
+			     vdev_phys, vdev_params_phys, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +30 -0
index 4a5ba98c1c0d..e5238b271493 100644
@@ -53,6 +53,8 @@
 #define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
 #define SMC_RMI_PDEV_SET_PUBKEY		SMC_RMI_CALL(0x017b)
 #define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
+#define SMC_RMI_VDEV_CREATE		SMC_RMI_CALL(0x0187)
+
 
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0
@@ -407,4 +409,32 @@ struct rmi_public_key_params {
 	};
 };
 
+enum rmi_vdev_state {
+	RMI_VDEV_READY,
+	RMI_VDEV_COMMUNICATING,
+	RMI_VDEV_STOPPING,
+	RMI_VDEV_STOPPED,
+	RMI_VDEV_ERROR,
+};
+
+#define MAX_VDEV_AUX_GRANULES	32
+
+struct rmi_vdev_params {
+	union {
+		struct {
+			u64 flags;
+			u64 vdev_id;
+			u64 tdi_id;
+			u64 num_aux;
+		};
+		u8 padding1[0x100];
+	};
+	union {	/* 0x100 */
+		struct {
+			unsigned long aux[MAX_VDEV_AUX_GRANULES];
+		};
+		u8 padding2[0x900];
+	};
+};
+
 #endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/Kconfig +1 -0
index a5b777f0d50e..52ed1cd5f06a 100644
@@ -6,6 +6,7 @@ config ARM_CCA_HOST
 	tristate "Arm CCA Host driver"
 	depends on ARM64
 	depends on PCI_TSM
+	depends on KVM
 	select KEYS
 	select X509_CERTIFICATE_PARSER
 	select TSM
drivers/virt/coco/arm-cca-host/arm-cca.c +30 -0
index c65b81f0706f..2da513f45974 100644
@@ -10,6 +10,8 @@
 #include <linux/pci.h>
 #include <linux/tsm.h>
 #include <linux/vmalloc.h>
+#include <linux/kvm_host.h>
+#include <linux/pci.h>
 
 #include "rmm-da.h"
 
@@ -221,11 +223,39 @@ static void cca_tsm_disconnect(struct pci_dev *pdev)
 	free_dev_communication_buffers(&dsc_pf0->comm_data);
 }
 
+static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_dev,
+				    struct kvm *kvm, u64 tdi_id)
+{
+	void *rmm_vdev;
+	struct cca_host_tdi *host_tdi __free(kfree) = NULL;
+	struct realm *realm = &kvm->arch.realm;
+
+	if (pdev->is_virtfn)
+		return ERR_PTR(-ENXIO);
+
+	if (!try_module_get(THIS_MODULE))
+		return ERR_PTR(-ENXIO);
+
+	host_tdi = kmalloc(sizeof(struct cca_host_tdi), GFP_KERNEL);
+	if (!host_tdi)
+		return ERR_PTR(-ENOMEM);
+
+	rmm_vdev = rme_create_vdev(realm, pdev, pf0_dev, tdi_id);
+	if (!IS_ERR_OR_NULL(rmm_vdev)) {
+		host_tdi->rmm_vdev = rmm_vdev;
+		return &no_free_ptr(host_tdi)->tdi;
+	}
+
+	module_put(THIS_MODULE);
+	return rmm_vdev;
+}
+
 static const struct pci_tsm_ops cca_pci_ops = {
 	.probe = cca_tsm_pci_probe,
 	.remove = cca_tsm_pci_remove,
 	.connect = cca_tsm_connect,
 	.disconnect = cca_tsm_disconnect,
+	.bind	= cca_tsm_bind,
 };
 
 static void cca_tsm_remove(void *tsm_core)
drivers/virt/coco/arm-cca-host/rmm-da.c +67 -0

76 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h +17 -0
index 03c3149b8a98..6d612ea3b87f 100644
@@ -45,6 +45,11 @@ struct cca_host_dsc_pf0 {
 	struct cache_object vca;
 };
 
+struct cca_host_tdi {
+	struct pci_tdi tdi;
+	void *rmm_vdev;
+};
+
 #define PDEV_COMMUNICATE	0x1
 static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
 {
@@ -71,7 +76,19 @@ static inline struct cca_host_comm_data *to_cca_comm_data(struct pci_dev *pdev)
 	return NULL;
 }
 
+static inline struct cca_host_tdi *to_cca_host_tdi(struct pci_dev *pdev)
+{
+	struct pci_tsm *tsm = pdev->tsm;
+
+	if (!tsm || !tsm->tdi)
+		return NULL;
+
+	return container_of(tsm->tdi, struct cca_host_tdi, tdi);
+}
+
 int rme_asign_device(struct pci_dev *pdev);
 void rme_unassign_device(struct pci_dev *pdev);
 int schedule_rme_ide_setup(struct pci_dev *pdev);
+void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,
+		      struct pci_dev *pf0_dev, u32 guest_rid);
 #endif
Signature · 1 line
2.43.0
author maintainer
#121 lore raw

RFC PATCH 21/38 coco: host: arm64: Add support for virtual device communication

In reply to Aneesh Kumar K.V (Arm)

Add support for vdev_communicate with RMM.

---
 arch/arm64/include/asm/rmi_cmds.h       | 22 ++++++++++++++++++++++
 arch/arm64/include/asm/rmi_smc.h        |  2 ++
 drivers/virt/coco/arm-cca-host/rmm-da.c | 21 +++++++++++++++++++--
 drivers/virt/coco/arm-cca-host/rmm-da.h |  2 ++
 4 files changed, 45 insertions(+), 2 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +22 -0
index 7d91f847069b..25197f47a0a9 100644
@@ -587,4 +587,26 @@ static inline unsigned long rmi_vdev_create(unsigned long rd,
 	return res.a0;
 }
 
+static inline unsigned long rmi_vdev_communicate(unsigned long pdev_phys,
+						 unsigned long vdev_phys,
+						 unsigned long vdev_comm_data_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_COMMUNICATE,
+			     pdev_phys, vdev_phys, vdev_comm_data_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_vdev_get_state(unsigned long vdev_phys, unsigned long *state)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_GET_STATE, vdev_phys, &res);
+
+	*state = res.a1;
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +2 -0
index e5238b271493..127dd0938604 100644
@@ -53,7 +53,9 @@
 #define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
 #define SMC_RMI_PDEV_SET_PUBKEY		SMC_RMI_CALL(0x017b)
 #define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
+#define SMC_RMI_VDEV_COMMUNICATE	SMC_RMI_CALL(0x0186)
 #define SMC_RMI_VDEV_CREATE		SMC_RMI_CALL(0x0187)
+#define SMC_RMI_VDEV_GET_STATE		SMC_RMI_CALL(0x0189)
 
 
 #define RMI_ABI_MAJOR_VERSION	1
drivers/virt/coco/arm-cca-host/rmm-da.c +19 -2
index 41314db1d568..8635f361bbe8 100644
@@ -207,8 +207,14 @@ static int __do_dev_communicate(int type, struct pci_tsm *tsm)
 	if (type == PDEV_COMMUNICATE)
 		ret = rmi_pdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
 					   virt_to_phys(comm_data->io_params));
-	else
-		ret = RMI_ERROR_INPUT;
+	else {
+		struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi);
+
+		ret = rmi_vdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
+					   virt_to_phys(host_tdi->rmm_vdev),
+					   virt_to_phys(comm_data->io_params));
+	}
+
 	if (ret != RMI_SUCCESS) {
 		pr_err("pdev communicate error\n");
 		return ret;
@@ -299,6 +305,12 @@ static int do_dev_communicate(int type, struct pci_tsm *tsm, int target_state)
 			ret = rmi_pdev_get_state(virt_to_phys(dsc_pf0->rmm_pdev),
 						 &state);
 			error_state = RMI_PDEV_ERROR;
+		} else {
+			struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi);
+
+			ret = rmi_vdev_get_state(virt_to_phys(host_tdi->rmm_vdev),
+						 &state);
+			error_state = RMI_VDEV_ERROR;
 		}
 		if (ret != 0) {
 			pr_err("Get dev state error\n");
@@ -584,3 +596,8 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,
 err_out:
 	return ERR_PTR(ret);
 }
+
+static int __maybe_unused do_vdev_communicate(struct pci_tsm *tsm, int target_state)
+{
+	return do_dev_communicate(VDEV_COMMUNICATE, tsm, target_state);
+}
drivers/virt/coco/arm-cca-host/rmm-da.h +2 -0
index 6d612ea3b87f..37a8f4dce68e 100644
@@ -51,6 +51,8 @@ struct cca_host_tdi {
 };
 
 #define PDEV_COMMUNICATE	0x1
+#define VDEV_COMMUNICATE	0x2
+
 static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
 {
 	struct pci_tsm *tsm = pdev->tsm;
Signature · 1 line
2.43.0
author maintainer
#124 lore raw

RFC PATCH 22/38 coco: host: arm64: Stop and destroy virtual device

In reply to Aneesh Kumar K.V (Arm)

Add support for vdev_stop and vdev_destroy.

---
 arch/arm64/include/asm/rmi_cmds.h        | 21 ++++++++
 arch/arm64/include/asm/rmi_smc.h         |  3 +-
 drivers/virt/coco/arm-cca-host/arm-cca.c | 10 ++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 61 +++++++++++++++++++++++-
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  2 +
 5 files changed, 95 insertions(+), 2 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +21 -0
index 25197f47a0a9..eb4f67eb6b01 100644
@@ -609,4 +609,25 @@ static inline unsigned long rmi_vdev_get_state(unsigned long vdev_phys, unsigned
 	return res.a0;
 }
 
+static inline unsigned long rmi_vdev_stop(unsigned long vdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_STOP, vdev_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_vdev_destroy(unsigned long rd,
+					     unsigned long pdev_phys,
+					     unsigned long vdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_DESTROY, rd, pdev_phys, vdev_phys, &res);
+
+	return res.a0;
+}
+
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +2 -1
index 127dd0938604..c6e16ab608e1 100644
@@ -55,8 +55,9 @@
 #define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
 #define SMC_RMI_VDEV_COMMUNICATE	SMC_RMI_CALL(0x0186)
 #define SMC_RMI_VDEV_CREATE		SMC_RMI_CALL(0x0187)
+#define SMC_RMI_VDEV_DESTROY		SMC_RMI_CALL(0x0188)
 #define SMC_RMI_VDEV_GET_STATE		SMC_RMI_CALL(0x0189)
-
+#define SMC_RMI_VDEV_STOP		SMC_RMI_CALL(0x018A)
 
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0
drivers/virt/coco/arm-cca-host/arm-cca.c +10 -0
index 2da513f45974..3792d7b5cb99 100644
@@ -250,12 +250,22 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_de
 	return rmm_vdev;
 }
 
+static void cca_tsm_unbind(struct pci_tdi *tdi)
+{
+	struct realm *realm = &tdi->kvm->arch.realm;
+
+	rme_unbind_vdev(realm, tdi->pdev, tdi->pdev->tsm->dsm_dev);
+
+	module_put(THIS_MODULE);
+}
+
 static const struct pci_tsm_ops cca_pci_ops = {
 	.probe = cca_tsm_pci_probe,
 	.remove = cca_tsm_pci_remove,
 	.connect = cca_tsm_connect,
 	.disconnect = cca_tsm_disconnect,
 	.bind	= cca_tsm_bind,
+	.unbind = cca_tsm_unbind,
 };
 
 static void cca_tsm_remove(void *tsm_core)
drivers/virt/coco/arm-cca-host/rmm-da.c +60 -1

67 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h +2 -0
index 37a8f4dce68e..6361f7403f95 100644
@@ -93,4 +93,6 @@ void rme_unassign_device(struct pci_dev *pdev);
 int schedule_rme_ide_setup(struct pci_dev *pdev);
 void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,
 		      struct pci_dev *pf0_dev, u32 guest_rid);
+void rme_unbind_vdev(struct realm *realm, struct pci_dev *pdev,
+		     struct pci_dev *pf0_dev);
 #endif
Signature · 1 line
2.43.0
author maintainer
#126 lore raw

RFC PATCH 23/38 coco: guest: arm64: Update arm CCA guest driver

In reply to Aneesh Kumar K.V (Arm)

This patch includes renaming changes to simplify the registration of a TSM backend in the next patch. There are no functional changes in this update.

---
 arch/arm64/include/asm/rsi.h                  |  2 +-
 arch/arm64/kernel/rsi.c                       |  2 +-
 drivers/virt/coco/arm-cca-guest/Makefile      |  3 ++
 .../{arm-cca-guest.c => arm-cca.c}            | 52 +++++++++----------
 4 files changed, 29 insertions(+), 30 deletions(-)
 rename drivers/virt/coco/arm-cca-guest/{arm-cca-guest.c => arm-cca.c} (86%)
arch/arm64/include/asm/rsi.h +1 -1
index b42aeac05340..26ef6143562b 100644
@@ -10,7 +10,7 @@
 #include <linux/jump_label.h>
 #include <asm/rsi_cmds.h>
 
-#define RSI_PDEV_NAME "arm-cca-dev"
+#define RSI_DEV_NAME "arm-rsi-dev"
 
 DECLARE_STATIC_KEY_FALSE(rsi_present);
 
arch/arm64/kernel/rsi.c +1 -1
index ce4778141ec7..bf9ea99e2aa1 100644
@@ -142,7 +142,7 @@ void __init arm64_rsi_init(void)
 }
 
 static struct platform_device rsi_dev = {
-	.name = RSI_PDEV_NAME,
+	.name = RSI_DEV_NAME,
 	.id = PLATFORM_DEVID_NONE
 };
 
drivers/virt/coco/arm-cca-guest/Makefile +3 -0
index 69eeba08e98a..609462ea9438 100644
@@ -1,2 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
+#
 obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest.o
+
+arm-cca-guest-$(CONFIG_TSM) +=  arm-cca.o
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +24 -28

82 lines not on this page · raw message

Signature · 1 line
2.43.0
author maintainer
#130 lore raw

RFC PATCH 24/38 arm64: CCA: Register guest tsm callback

In reply to Aneesh Kumar K.V (Arm)

Register the TSM callback if the DA feature is supported by RSI.

Additionally, adjust the build order so that the TSM class is created before the arm-cca-guest driver initialization.

---
 arch/arm64/include/asm/rsi.h              |  3 +
 arch/arm64/include/asm/rsi_cmds.h         | 18 ++++++
 arch/arm64/include/asm/rsi_smc.h          |  1 +
 arch/arm64/kernel/rsi.c                   | 24 ++++++--
 drivers/virt/coco/Makefile                |  2 +-
 drivers/virt/coco/arm-cca-guest/Kconfig   |  8 ++-
 drivers/virt/coco/arm-cca-guest/arm-cca.c | 71 ++++++++++++++++++++++-
 drivers/virt/coco/arm-cca-guest/rsi-da.h  | 27 +++++++++
 8 files changed, 144 insertions(+), 10 deletions(-)
 create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.h
arch/arm64/include/asm/rsi.h +3 -0
index 26ef6143562b..35dfbba4767b 100644
@@ -67,4 +67,7 @@ static inline int rsi_set_memory_range_shared(phys_addr_t start,
 	return rsi_set_memory_range(start, end, RSI_RIPAS_EMPTY,
 				    RSI_CHANGE_DESTROYED);
 }
+
+bool rsi_has_da_feature(void);
+
 #endif /* __ASM_RSI_H_ */
arch/arm64/include/asm/rsi_cmds.h +18 -0
index 2c8763876dfb..d4834baeef1b 100644
@@ -159,4 +159,22 @@ static inline unsigned long rsi_attestation_token_continue(phys_addr_t granule,
 	return res.a0;
 }
 
+/**
+ * rsi_features() - Read feature register
+ * @index: Feature register index
+ * @out: Feature register value is written to this pointer
+ *
+ * Return: RSI return code
+ */
+static inline int rsi_features(unsigned long index, unsigned long *out)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_FEATURES, index, &res);
+
+	if (out)
+		*out = res.a1;
+	return res.a0;
+}
+
 #endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +1 -0
index 6cb070eca9e9..8e486cdef9eb 100644
@@ -53,6 +53,7 @@
  */
 #define SMC_RSI_ABI_VERSION	SMC_RSI_FID(0x190)
 
+#define RSI_FEATURE_REGISTER_0_DA		BIT(0)
 /*
  * Read feature register.
  *
arch/arm64/kernel/rsi.c +18 -6
index bf9ea99e2aa1..ef06c083990a 100644
@@ -15,6 +15,7 @@
 #include <asm/rsi.h>
 
 static struct realm_config config;
+static unsigned long rsi_feat_reg0;
 
 unsigned long prot_ns_shared;
 EXPORT_SYMBOL(prot_ns_shared);
@@ -22,6 +23,12 @@ EXPORT_SYMBOL(prot_ns_shared);
 DEFINE_STATIC_KEY_FALSE_RO(rsi_present);
 EXPORT_SYMBOL(rsi_present);
 
+bool rsi_has_da_feature(void)
+{
+	return !!u64_get_bits(rsi_feat_reg0, RSI_FEATURE_REGISTER_0_DA);
+}
+EXPORT_SYMBOL_GPL(rsi_has_da_feature);
+
 bool cc_platform_has(enum cc_attr attr)
 {
 	switch (attr) {
@@ -128,6 +135,10 @@ void __init arm64_rsi_init(void)
 		return;
 	if (WARN_ON(rsi_get_realm_config(&config)))
 		return;
+
+	if (WARN_ON(rsi_features(0, &rsi_feat_reg0)))
+		return;
+
 	prot_ns_shared = BIT(config.ipa_bits - 1);
 
 	if (arm64_ioremap_prot_hook_register(realm_ioremap_hook))
@@ -141,17 +152,18 @@ void __init arm64_rsi_init(void)
 	static_branch_enable(&rsi_present);
 }
 
-static struct platform_device rsi_dev = {
+static struct platform_device cca_guest_dev = {
 	.name = RSI_DEV_NAME,
 	.id = PLATFORM_DEVID_NONE
 };
 
-static int __init arm64_create_dummy_rsi_dev(void)
+static int __init arm64_create_cca_guest_dev(void)
 {
-	if (is_realm_world() &&
-	    platform_device_register(&rsi_dev))
-		pr_err("failed to register rsi platform device\n");
+	if (is_realm_world()) {
+		if (!platform_device_register(&cca_guest_dev))
+			pr_info("CCA guest platform device registered.\n");
+	}
 	return 0;
 }
 
-arch_initcall(arm64_create_dummy_rsi_dev)
+device_initcall(arm64_create_cca_guest_dev)
drivers/virt/coco/Makefile +1 -1
index d0a859dd9eaf..4264ee367b3b 100644
@@ -7,8 +7,8 @@ obj-$(CONFIG_EFI_SECRET)	+= efi_secret/
 obj-$(CONFIG_ARM_PKVM_GUEST)	+= pkvm-guest/
 obj-$(CONFIG_SEV_GUEST)		+= sev-guest/
 obj-$(CONFIG_INTEL_TDX_GUEST)	+= tdx-guest/
-obj-$(CONFIG_ARM_CCA_GUEST)	+= arm-cca-guest/
 
 obj-$(CONFIG_TSM) 		+= tsm-core.o
 obj-y				+= guest/
+obj-$(CONFIG_ARM_CCA_GUEST)	+= arm-cca-guest/
 obj-$(CONFIG_ARM_CCA_HOST)	+= arm-cca-host/
drivers/virt/coco/arm-cca-guest/Kconfig +7 -1
index 3f0f013f03f1..410d9c3fb2b3 100644
@@ -1,10 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
 config ARM_CCA_GUEST
 	tristate "Arm CCA Guest driver"
 	depends on ARM64
+	depends on PCI_TSM
 	select TSM_REPORTS
+	select TSM
 	help
-	  The driver provides userspace interface to request and
+	  The driver provides userspace interface to request an
 	  attestation report from the Realm Management Monitor(RMM).
+	  If the DA feature is supported, it also register with TSM framework.
 
 	  If you choose 'M' here, this module will be called
 	  arm-cca-guest.
drivers/virt/coco/arm-cca-guest/arm-cca.c +69 -2

98 lines not on this page · raw message

drivers/virt/coco/arm-cca-guest/rsi-da.h new file +27 -0
new file mode 100644
index 000000000000..8a4d5f1b0263
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2025 ARM Ltd.
+ */
+
+#ifndef RSI_DA_H_
+#define RSI_DA_H_
+
+#include <linux/pci.h>
+#include <linux/pci-tsm.h>
+#include <asm/rsi_smc.h>
+
+
+struct cca_guest_dsc {
+	struct pci_tsm_pf0 pci;
+};
+
+static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
+{
+	struct pci_tsm *tsm = pdev->tsm;
+
+	if (!tsm)
+		return NULL;
+	return container_of(tsm, struct cca_guest_dsc, pci.tsm);
+}
+
+#endif
Signature · 1 line
2.43.0
author maintainer
#133 lore raw

RFC PATCH 25/38 cca: guest: arm64: Realm device lock support

In reply to Aneesh Kumar K.V (Arm)

Writing 1 to 'tsm/lock' will initiate the TDISP lock sequence.

---
 arch/arm64/include/asm/rsi_cmds.h         | 32 +++++++++++++-
 arch/arm64/include/asm/rsi_smc.h          |  5 +++
 drivers/virt/coco/arm-cca-guest/Makefile  |  2 +-
 drivers/virt/coco/arm-cca-guest/arm-cca.c | 18 ++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.c  | 52 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h  |  3 +-
 6 files changed, 108 insertions(+), 4 deletions(-)
 create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.c
arch/arm64/include/asm/rsi_cmds.h +30 -2
index d4834baeef1b..b9c4b8ff5631 100644
@@ -172,8 +172,36 @@ static inline int rsi_features(unsigned long index, unsigned long *out)
 
 	arm_smccc_1_1_invoke(SMC_RSI_FEATURES, index, &res);
 
-	if (out)
-		*out = res.a1;
+	*out = res.a1;
+	return res.a0;
+}
+
+static inline unsigned long rsi_rdev_get_instance_id(unsigned long vdev_id, unsigned long *inst_id)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_GET_INSTANCE_ID, vdev_id, &res);
+
+	*inst_id = res.a1;
+	return res.a0;
+}
+
+static inline unsigned long __rsi_rdev_lock(unsigned long vdev_id, unsigned long inst_id)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_LOCK, vdev_id, inst_id, &res);
+
+	return res.a0;
+}
+
+
+static inline unsigned long rsi_rdev_continue(unsigned long vdev_id, unsigned long inst_id)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_CONTINUE, vdev_id, inst_id, &res);
+
 	return res.a0;
 }
 
arch/arm64/include/asm/rsi_smc.h +5 -0
index 8e486cdef9eb..44b583ab6d67 100644
@@ -191,4 +191,9 @@ struct realm_config {
  */
 #define SMC_RSI_HOST_CALL			SMC_RSI_FID(0x199)
 
+#define SMC_RSI_RDEV_GET_INSTANCE_ID		SMC_RSI_FID(0x19c)
+#define SMC_RSI_RDEV_CONTINUE			SMC_RSI_FID(0x1a4)
+
+#define SMC_RSI_RDEV_LOCK			SMC_RSI_FID(0x1a9)
+
 #endif /* __ASM_RSI_SMC_H_ */
drivers/virt/coco/arm-cca-guest/Makefile +1 -1
index 609462ea9438..341c7b37d610 100644
@@ -2,4 +2,4 @@
 #
 obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest.o
 
-arm-cca-guest-$(CONFIG_TSM) +=  arm-cca.o
+arm-cca-guest-$(CONFIG_TSM) +=  arm-cca.o rsi-da.o
drivers/virt/coco/arm-cca-guest/arm-cca.c +18 -0
index 3adbbd67e06e..2c0190bcb2a9 100644
@@ -220,9 +220,27 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)
 	kfree(cca_dsc);
 }
 
+static int cca_tsm_lock(struct pci_dev *pdev)
+{
+	unsigned long ret;
+
+	ret = rsi_device_lock(pdev);
+	if (ret) {
+		pci_err(pdev, "failed to lock the device (%lu)\n", ret);
+		return -EIO;
+	}
+	return 0;
+}
+
+static void cca_tsm_unlock(struct pci_dev *pdev)
+{
+}
+
 static const struct pci_tsm_ops cca_pci_ops = {
 	.probe = cca_tsm_pci_probe,
 	.remove = cca_tsm_pci_remove,
+	.lock = cca_tsm_lock,
+	.unlock = cca_tsm_unlock,
 };
 
 static void cca_tsm_unregister(void *tsm)
drivers/virt/coco/arm-cca-guest/rsi-da.c new file +52 -0

52 lines not on this page · raw message

drivers/virt/coco/arm-cca-guest/rsi-da.h +2 -1
index 8a4d5f1b0263..f12430c7d792 100644
@@ -10,9 +10,9 @@
 #include <linux/pci-tsm.h>
 #include <asm/rsi_smc.h>
 
-
 struct cca_guest_dsc {
 	struct pci_tsm_pf0 pci;
+	unsigned long instance_id;
 };
 
 static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
@@ -24,4 +24,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
 	return container_of(tsm, struct cca_guest_dsc, pci.tsm);
 }
 
+int rsi_device_lock(struct pci_dev *pdev);
 #endif
Signature · 1 line
2.43.0
author maintainer
#135 lore raw

RFC PATCH 26/38 KVM: arm64: Add exit handler related to device assignment

In reply to Aneesh Kumar K.V (Arm)

Different RSI calls related to DA result in REC exits. Add a facility to register handlers for handling these REC exits.

---
 arch/arm64/include/asm/kvm_rme.h |  3 ++
 arch/arm64/include/asm/rmi_smc.h | 14 +++++++-
 arch/arm64/kvm/rme-exit.c        | 60 ++++++++++++++++++++++++++++++++
 3 files changed, 76 insertions(+), 1 deletion(-)
arch/arm64/include/asm/kvm_rme.h +3 -0
index e954bb95dc86..370d056222e8 100644
@@ -136,4 +136,7 @@ static inline bool kvm_realm_is_private_address(struct realm *realm,
 	return !(addr & BIT(realm->ia_bits - 1));
 }
 
+extern int (*realm_exit_vdev_req_handler)(struct realm_rec *rec);
+extern int (*realm_exit_vdev_comm_handler)(struct realm_rec *rec);
+extern int (*realm_exit_dev_mem_map_handler)(struct realm_rec *rec);
 #endif /* __ASM_KVM_RME_H */
arch/arm64/include/asm/rmi_smc.h +13 -1
index c6e16ab608e1..a5ef68b62bc0 100644
@@ -219,6 +219,9 @@ struct rec_enter {
 #define RMI_EXIT_RIPAS_CHANGE		0x04
 #define RMI_EXIT_HOST_CALL		0x05
 #define RMI_EXIT_SERROR			0x06
+#define RMI_EXIT_VDEV_REQUEST		0x08
+#define RMI_EXIT_VDEV_COMM		0x09
+#define RMI_EXIT_DEV_MEM_MAP		0x0a
 
 struct rec_exit {
 	union { /* 0x000 */
@@ -264,7 +267,16 @@ struct rec_exit {
 		u8 padding5[0x100];
 	};
 	union { /* 0x600 */
-		u16 imm;
+		struct {
+			u16 imm;
+			u8 padding[6];
+			u64 plane;
+			u64 vdev;
+			u64 vdev_action;
+			u64 dev_mem_base;
+			u64 dev_mem_top;
+			u64 dev_mem_pa;
+		};
 		u8 padding6[0x100];
 	};
 	union { /* 0x700 */
arch/arm64/kvm/rme-exit.c +60 -0

72 lines not on this page · raw message

Signature · 1 line
2.43.0
author maintainer
#137 lore raw

RFC PATCH 27/38 coco: host: arm64: add RSI_RDEV_GET_INSTANCE_ID related exit handler

In reply to Aneesh Kumar K.V (Arm)

Mapping the VDEV object that matches a specified virtual device ID results in a REC exit, which is handled by the VDEV request exit handler.

---
 arch/arm64/include/asm/rmi_cmds.h        |  8 +++++
 arch/arm64/include/asm/rmi_smc.h         |  9 ++++--
 drivers/virt/coco/arm-cca-host/arm-cca.c |  8 ++++-
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 39 ++++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  2 ++
 5 files changed, 63 insertions(+), 3 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +8 -0
index eb4f67eb6b01..fcf6b319e953 100644
@@ -629,5 +629,13 @@ static inline unsigned long rmi_vdev_destroy(unsigned long rd,
 	return res.a0;
 }
 
+static inline unsigned long rmi_vdev_complete(unsigned long rec_phys, unsigned long vdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_COMPLETE, rec_phys, vdev_phys, &res);
+
+	return res.a0;
+}
 
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +7 -2
index a5ef68b62bc0..6b23afa070d1 100644
@@ -57,7 +57,8 @@
 #define SMC_RMI_VDEV_CREATE		SMC_RMI_CALL(0x0187)
 #define SMC_RMI_VDEV_DESTROY		SMC_RMI_CALL(0x0188)
 #define SMC_RMI_VDEV_GET_STATE		SMC_RMI_CALL(0x0189)
-#define SMC_RMI_VDEV_STOP		SMC_RMI_CALL(0x018A)
+#define SMC_RMI_VDEV_STOP		SMC_RMI_CALL(0x018a)
+#define SMC_RMI_VDEV_COMPLETE		SMC_RMI_CALL(0x018e)
 
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0
@@ -262,7 +263,11 @@ struct rec_exit {
 		struct {
 			u64 ripas_base;
 			u64 ripas_top;
-			u64 ripas_value;
+			u8 ripas_value;
+			u8 padding8[15];
+			u64 s2ap_base;
+			u64 s2ap_top;
+			u64 vdev_id;
 		};
 		u8 padding5[0x100];
 	};
drivers/virt/coco/arm-cca-host/arm-cca.c +7 -1
index 3792d7b5cb99..837bd10ccd47 100644
@@ -275,13 +275,19 @@ static void cca_tsm_remove(void *tsm_core)
 
 static int cca_tsm_probe(struct platform_device *pdev)
 {
+	int rc;
 	struct tsm_core_dev *tsm_core;
 
 	tsm_core = tsm_register(&pdev->dev, NULL, &cca_pci_ops);
 	if (IS_ERR(tsm_core))
 		return PTR_ERR(tsm_core);
 
-	return devm_add_action_or_reset(&pdev->dev, cca_tsm_remove, tsm_core);
+	rc = devm_add_action_or_reset(&pdev->dev, cca_tsm_remove, tsm_core);
+	if (rc)
+		return rc;
+
+	rme_register_exit_handlers();
+	return 0;
 }
 
 static const struct platform_device_id arm_cca_host_id_table[] = {
drivers/virt/coco/arm-cca-host/rmm-da.c +39 -0
index 53072610fa67..d4f1da590b90 100644
@@ -660,3 +660,42 @@ void rme_unbind_vdev(struct realm *realm, struct pci_dev *pdev, struct pci_dev *
 		return;
 	}
 }
+
+static struct pci_tsm *find_pci_tsm_from_vdev_id(unsigned long vdev_id)
+{
+	struct pci_dev *pdev = NULL;
+	struct cca_host_tdi *host_tdi;
+
+	for_each_pci_dev(pdev) {
+		host_tdi = to_cca_host_tdi(pdev);
+		if (!host_tdi)
+			continue;
+		if (host_tdi->vdev_id == vdev_id)
+			return pdev->tsm;
+	}
+	return NULL;
+}
+
+static int rme_exit_vdev_req_handler(struct realm_rec *rec)
+{
+	struct cca_host_tdi *host_tdi = NULL;
+	unsigned long vdev_id = rec->run->exit.vdev_id;
+	struct pci_tsm *tsm = find_pci_tsm_from_vdev_id(vdev_id);
+	phys_addr_t rec_phys = virt_to_phys(rec->rec_page);
+
+	if (tsm)
+		host_tdi = to_cca_host_tdi(tsm->pdev);
+
+	if (host_tdi)
+		rmi_vdev_complete(rec_phys, virt_to_phys(host_tdi->rmm_vdev));
+	/*
+	 * Return back to the guest without calling vdev complete.
+	 * The Realm will treat that as an error.
+	 */
+	return 1;
+}
+
+void rme_register_exit_handlers(void)
+{
+	realm_exit_vdev_req_handler = rme_exit_vdev_req_handler;
+}
drivers/virt/coco/arm-cca-host/rmm-da.h +2 -0
index 6361f7403f95..7f51b611467b 100644
@@ -48,6 +48,7 @@ struct cca_host_dsc_pf0 {
 struct cca_host_tdi {
 	struct pci_tdi tdi;
 	void *rmm_vdev;
+	unsigned long vdev_id;
 };
 
 #define PDEV_COMMUNICATE	0x1
@@ -95,4 +96,5 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,
 		      struct pci_dev *pf0_dev, u32 guest_rid);
 void rme_unbind_vdev(struct realm *realm, struct pci_dev *pdev,
 		     struct pci_dev *pf0_dev);
+void rme_register_exit_handlers(void);
 #endif
Signature · 1 line
2.43.0
author maintainer
#138 lore raw

RFC PATCH 28/38 coco: host: arm64: Add support for device communication exit handler

In reply to Aneesh Kumar K.V (Arm)

Different RSI calls that require device communication result in a REC exit, which is handled by the device communication exit handler.

---
 arch/arm64/include/asm/rmi_smc.h         |  6 ++
 drivers/virt/coco/arm-cca-host/arm-cca.c |  1 +
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 75 ++++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  4 ++
 4 files changed, 86 insertions(+)
arch/arm64/include/asm/rmi_smc.h +6 -0
index 6b23afa070d1..7073eccaec5f 100644
@@ -457,4 +457,10 @@ struct rmi_vdev_params {
 	};
 };
 
+#define RMI_VDEV_ACTION_GET_INTERFACE_REPORT	0x0
+#define RMI_VDEV_ACTION_GET_MEASUREMENTS	0x1
+#define RMI_VDEV_ACTION_LOCK			0x2
+#define RMI_VDEV_ACTION_START			0x3
+#define RMI_VDEV_ACTION_STOP			0x4
+
 #endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/arm-cca.c +1 -0
index 837bd10ccd47..be1296fb1bf2 100644
@@ -243,6 +243,7 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_de
 	rmm_vdev = rme_create_vdev(realm, pdev, pf0_dev, tdi_id);
 	if (!IS_ERR_OR_NULL(rmm_vdev)) {
 		host_tdi->rmm_vdev = rmm_vdev;
+		host_tdi->vdev_id = tdi_id;
 		return &no_free_ptr(host_tdi)->tdi;
 	}
 
drivers/virt/coco/arm-cca-host/rmm-da.c +75 -0

98 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h +4 -0
index 7f51b611467b..cebddab8464d 100644
@@ -22,6 +22,8 @@ struct cca_host_comm_data {
 	void *resp_buff;
 	void *req_buff;
 	struct rmi_dev_comm_data *io_params;
+
+	bool vdev_comm_active;
 };
 
 struct cca_host_dsc_pf0 {
@@ -43,6 +45,8 @@ struct cca_host_dsc_pf0 {
 		bool valid;
 	} cert_chain;
 	struct cache_object vca;
+	struct cache_object interface_report;
+	struct cache_object measurements;
 };
 
 struct cca_host_tdi {
Signature · 1 line
2.43.0
author maintainer
#139 lore raw

RFC PATCH 29/38 coco: guest: arm64: Add support for collecting interface reports

In reply to Aneesh Kumar K.V (Arm)

Support collecting interface reports using RSI calls.

---
 arch/arm64/include/asm/rsi_cmds.h        | 14 ++++++++++
 arch/arm64/include/asm/rsi_smc.h         |  2 ++
 drivers/virt/coco/arm-cca-guest/rsi-da.c | 34 ++++++++++++++++++++++++
 3 files changed, 50 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +14 -0
index b9c4b8ff5631..1d76f7d37cb6 100644
@@ -205,4 +205,18 @@ static inline unsigned long rsi_rdev_continue(unsigned long vdev_id, unsigned lo
 	return res.a0;
 }
 
+static inline unsigned long __rsi_rdev_get_interface_report(unsigned long vdev_id,
+							  unsigned long inst_id,
+							  unsigned long version_max,
+							  unsigned long *version)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_GET_INTERFACE_REPORT,
+			     vdev_id, inst_id, version_max, &res);
+
+	*version = res.a1;
+	return res.a0;
+}
+
 #endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +2 -0
index 44b583ab6d67..6afcccee2ae7 100644
@@ -194,6 +194,8 @@ struct realm_config {
 #define SMC_RSI_RDEV_GET_INSTANCE_ID		SMC_RSI_FID(0x19c)
 #define SMC_RSI_RDEV_CONTINUE			SMC_RSI_FID(0x1a4)
 
+#define SMC_RSI_RDEV_GET_INTERFACE_REPORT	SMC_RSI_FID(0x1a6)
+
 #define SMC_RSI_RDEV_LOCK			SMC_RSI_FID(0x1a9)
 
 #endif /* __ASM_RSI_SMC_H_ */
drivers/virt/coco/arm-cca-guest/rsi-da.c +34 -0
index 097cf52ee199..28ec946df1e2 100644
@@ -29,9 +29,31 @@ static inline unsigned long rsi_rdev_lock(struct pci_dev *pdev,
 	return RSI_SUCCESS;
 }
 
+static inline unsigned long
+rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id,
+			      unsigned long inst_id, unsigned long version_max,
+			      unsigned long *version)
+{
+	unsigned long ret;
+
+	ret = __rsi_rdev_get_interface_report(vdev_id, inst_id, version_max, version);
+	if (ret != RSI_SUCCESS)
+		return ret;
+
+	do {
+		ret = rsi_rdev_continue(vdev_id, inst_id);
+	} while (ret == RSI_INCOMPLETE);
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to communicate with the device (%lu)\n", ret);
+		return ret;
+	}
+	return RSI_SUCCESS;
+}
+
 int rsi_device_lock(struct pci_dev *pdev)
 {
 	unsigned long ret;
+	unsigned long tdisp_version;
 	struct cca_guest_dsc *dsm = to_cca_guest_dsc(pdev);
 	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
 		PCI_DEVID(pdev->bus->number, pdev->devfn);
@@ -48,5 +70,17 @@ int rsi_device_lock(struct pci_dev *pdev)
 		return -EIO;
 	}
 
+	ret = rsi_rdev_get_interface_report(pdev, vdev_id, dsm->instance_id,
+					   PCI_TDISP_MESSAGE_VERSION_10, &tdisp_version);
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to get interface report (%lu)\n", ret);
+		return -EIO;
+	}
+
+	if (tdisp_version != PCI_TDISP_MESSAGE_VERSION_10) {
+		pci_err(pdev, "unknown TDISP version (%lu)\n", tdisp_version);
+		return -EOPNOTSUPP;
+	}
+
 	return ret;
 }
Signature · 1 line
2.43.0
author maintainer
#140 lore raw

RFC PATCH 30/38 coco: host: arm64: Add support for realm host interface (RHI)

In reply to Aneesh Kumar K.V (Arm)

Device assignment-related RHI calls result in a REC exit, which is handled by the tsm guest_request callback.

---
 arch/arm64/include/asm/rhi.h             | 32 ++++++++++
 drivers/virt/coco/arm-cca-host/arm-cca.c | 68 ++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 81 ++++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  4 ++
 include/linux/tsm.h                      |  3 +
 5 files changed, 188 insertions(+)
 create mode 100644 arch/arm64/include/asm/rhi.h
arch/arm64/include/asm/rhi.h new file +32 -0
new file mode 100644
index 000000000000..d3c22e582678
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2024 ARM Ltd.
+ */
+
+#ifndef __ASM_RHI_H_
+#define __ASM_RHI_H_
+
+#define SMC_RHI_CALL(func)				\
+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,		\
+			   ARM_SMCCC_SMC_64,		\
+			   ARM_SMCCC_OWNER_STANDARD_HYP,\
+			   (func))
+
+
+#define RHI_DA_FEATURES			SMC_RHI_CALL(0x004d)
+#define RHI_DA_OBJECT_SIZE		SMC_RHI_CALL(0x004e)
+#define RHI_DA_OBJECT_READ		SMC_RHI_CALL(0x004f)
+
+#define RHI_DA_OBJECT_CERTIFICATE		0x1
+#define RHI_DA_OBJECT_MEASUREMENT		0x2
+#define RHI_DA_OBJECT_INTERFACE_REPORT		0x3
+#define RHI_DA_OBJECT_VCA			0x4
+
+
+#define RHI_DA_SUCCESS				0x1
+#define RHI_ERROR_INVALID_VDEV_ID		0x2
+#define RHI_ERROR_INVALID_DA_OBJECT_TYPE	0x3
+#define RHI_ERROR_DATA_NOT_AVAILABLE		0x4
+#define RHI_ERROR_INVALID_OFFSET		0x5
+#define RHI_ERROR_INVALID_ADDR			0x6
+#endif
drivers/virt/coco/arm-cca-host/arm-cca.c +68 -0

80 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.c +81 -0

94 lines not on this page · raw message

drivers/virt/coco/arm-cca-host/rmm-da.h +4 -0
index cebddab8464d..457660ff3b69 100644
@@ -10,6 +10,7 @@
 #include <linux/pci-ide.h>
 #include <linux/pci-tsm.h>
 #include <asm/rmi_smc.h>
+#include <asm/rhi.h>
 
 #define MAX_CACHE_OBJ_SIZE	4096
 struct cache_object {
@@ -101,4 +102,7 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,
 void rme_unbind_vdev(struct realm *realm, struct pci_dev *pdev,
 		     struct pci_dev *pf0_dev);
 void rme_register_exit_handlers(void);
+int rme_get_da_object_size(struct pci_dev *pdev, int type);
+int rme_da_object_read(struct pci_dev *pdev, int type, unsigned long offset,
+		       unsigned long max_len, void __user *user_buf);
 #endif
include/linux/tsm.h +3 -0
index 497a3b4df5a0..e82046b0c7fa 100644
@@ -145,5 +145,8 @@ struct tsm_guest_req_info {
 	size_t resp_len;
 };
 
+#define ARM_CCA_DA_OBJECT_SIZE 0x1
+#define ARM_CCA_DA_OBJECT_READ 0x2
+
 int tsm_guest_req(struct device *dev, struct tsm_guest_req_info *info);
 #endif /* __TSM_H */
Signature · 1 line
2.43.0
author maintainer
#142 lore raw

RFC PATCH 31/38 coco: guest: arm64: Add support for fetching interface report and certificate chain from host

In reply to Aneesh Kumar K.V (Arm)

Fetch interface report and certificate chain from the host using RHI calls.

---
 arch/arm64/include/asm/rsi_cmds.h        |   9 ++
 arch/arm64/include/asm/rsi_smc.h         |   6 ++
 drivers/virt/coco/arm-cca-guest/rsi-da.c | 131 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h |   5 +
 4 files changed, 151 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +9 -0
index 1d76f7d37cb6..18fc4e1ce577 100644
@@ -219,4 +219,13 @@ static inline unsigned long __rsi_rdev_get_interface_report(unsigned long vdev_i
 	return res.a0;
 }
 
+static inline unsigned long rsi_host_call(phys_addr_t addr)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_HOST_CALL, addr, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +6 -0
index 6afcccee2ae7..1d762fe3777b 100644
@@ -183,6 +183,12 @@ struct realm_config {
  */
 #define SMC_RSI_IPA_STATE_GET			SMC_RSI_FID(0x198)
 
+struct rsi_host_call {
+	u16 imm;
+	u8 padding[6];
+	u64 gprs[31];
+};
+
 /*
  * Make a Host call.
  *
drivers/virt/coco/arm-cca-guest/rsi-da.c +131 -0

148 lines not on this page · raw message

drivers/virt/coco/arm-cca-guest/rsi-da.h +5 -0
index f12430c7d792..bd565785ff4b 100644
@@ -9,10 +9,15 @@
 #include <linux/pci.h>
 #include <linux/pci-tsm.h>
 #include <asm/rsi_smc.h>
+#include <asm/rhi.h>
 
 struct cca_guest_dsc {
 	struct pci_tsm_pf0 pci;
 	unsigned long instance_id;
+	void *interface_report;
+	int interface_report_size;
+	void *certificate;
+	int certificate_size;
 };
 
 static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
Signature · 1 line
2.43.0
author maintainer
#144 lore raw

RFC PATCH 32/38 coco: guest: arm64: Add support for guest initiated TDI bind/unbind

In reply to Aneesh Kumar K.V (Arm)

Add RHI for VDEV_SET_TDI_STATE

Note: This is not part of RHI spec. This is a POC implementation and will be later switced to correct interface defined by RHI.

---
 arch/arm64/include/asm/rhi.h              |  7 +++++
 arch/arm64/kernel/Makefile                |  2 +-
 arch/arm64/kernel/rhi.c                   | 35 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/arm-cca.c | 22 ++++++++++++--
 drivers/virt/coco/arm-cca-host/arm-cca.c  |  8 ++++--
 5 files changed, 69 insertions(+), 5 deletions(-)
 create mode 100644 arch/arm64/kernel/rhi.c
arch/arm64/include/asm/rhi.h +7 -0
index d3c22e582678..993b4b15b057 100644
@@ -16,6 +16,7 @@
 #define RHI_DA_FEATURES			SMC_RHI_CALL(0x004d)
 #define RHI_DA_OBJECT_SIZE		SMC_RHI_CALL(0x004e)
 #define RHI_DA_OBJECT_READ		SMC_RHI_CALL(0x004f)
+#define RHI_DA_VDEV_SET_TDI_STATE	SMC_RHI_CALL(0x0052)
 
 #define RHI_DA_OBJECT_CERTIFICATE		0x1
 #define RHI_DA_OBJECT_MEASUREMENT		0x2
@@ -29,4 +30,10 @@
 #define RHI_ERROR_DATA_NOT_AVAILABLE		0x4
 #define RHI_ERROR_INVALID_OFFSET		0x5
 #define RHI_ERROR_INVALID_ADDR			0x6
+
+#define RHI_DA_TDI_CONFIG_UNLOCKED		0x0
+#define RHI_DA_TDI_CONFIG_LOCKED		0x1
+#define RHI_DA_TDI_CONFIG_RUN			0x2
+long rhi_da_vdev_set_tdi_state(unsigned long vdev_id, unsigned long target_state);
+
 #endif
arch/arm64/kernel/Makefile +1 -1
index a2faf0049dab..dde8fa78852c 100644
@@ -34,7 +34,7 @@ obj-y			:= debug-monitors.o entry.o irq.o fpsimd.o		\
 			   cpufeature.o alternative.o cacheinfo.o		\
 			   smp.o smp_spin_table.o topology.o smccc-call.o	\
 			   syscall.o proton-pack.o idle.o patching.o pi/	\
-			   rsi.o jump_label.o
+			   rsi.o jump_label.o rhi.o
 
 obj-$(CONFIG_COMPAT)			+= sys32.o signal32.o			\
 					   sys_compat.o
arch/arm64/kernel/rhi.c new file +35 -0
new file mode 100644
index 000000000000..3685b50c2e94
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 ARM Ltd.
+ */
+
+#include <asm/memory.h>
+#include <asm/string.h>
+#include <asm/rsi.h>
+#include <asm/rhi.h>
+
+#include <linux/slab.h>
+
+long rhi_da_vdev_set_tdi_state(unsigned long guest_rid, unsigned long target_state)
+{
+	long ret;
+	struct rsi_host_call *rhi_call;
+
+	rhi_call = kmalloc(sizeof(struct rsi_host_call), GFP_KERNEL);
+	if (!rhi_call)
+		return -ENOMEM;
+
+	rhi_call->imm = 0;
+	rhi_call->gprs[0] = RHI_DA_VDEV_SET_TDI_STATE;
+	rhi_call->gprs[1] = guest_rid;
+	rhi_call->gprs[2] = target_state;
+
+	ret = rsi_host_call(virt_to_phys(rhi_call));
+	if (ret != RSI_SUCCESS)
+		ret =  -EIO;
+	else
+		ret = rhi_call->gprs[0];
+
+	kfree(rhi_call);
+	return ret;
+}
drivers/virt/coco/arm-cca-guest/arm-cca.c +20 -2
index 2c0190bcb2a9..de70fba09e92 100644
@@ -222,11 +222,20 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)
 
 static int cca_tsm_lock(struct pci_dev *pdev)
 {
-	unsigned long ret;
+	long ret;
+	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
+		PCI_DEVID(pdev->bus->number, pdev->devfn);
 
+	ret = rhi_da_vdev_set_tdi_state(vdev_id, RHI_DA_TDI_CONFIG_LOCKED);
+	if (ret) {
+		pci_err(pdev, "failed to TSM bind the device (%ld)\n", ret);
+		return -EIO;
+	}
+
+	/* This will be done by above rhi in later spec */
 	ret = rsi_device_lock(pdev);
 	if (ret) {
-		pci_err(pdev, "failed to lock the device (%lu)\n", ret);
+		pci_err(pdev, "failed to lock the device (%ld)\n", ret);
 		return -EIO;
 	}
 	return 0;
@@ -234,6 +243,15 @@ static int cca_tsm_lock(struct pci_dev *pdev)
 
 static void cca_tsm_unlock(struct pci_dev *pdev)
 {
+	long ret;
+	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
+		PCI_DEVID(pdev->bus->number, pdev->devfn);
+
+	ret = rhi_da_vdev_set_tdi_state(vdev_id, RHI_DA_TDI_CONFIG_UNLOCKED);
+	if (ret) {
+		pci_err(pdev, "failed to TSM unbind the device (%ld)\n", ret);
+		return;
+	}
 }
 
 static const struct pci_tsm_ops cca_pci_ops = {
drivers/virt/coco/arm-cca-host/arm-cca.c +6 -2
index 0807fcf8d222..18d0a627baa4 100644
@@ -254,9 +254,13 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_de
 static void cca_tsm_unbind(struct pci_tdi *tdi)
 {
 	struct realm *realm = &tdi->kvm->arch.realm;
-
+	/*
+	 * FIXME!!
+	 * All the related DEV RIPAS regions should be unmapped by now.
+	 * For now we handle them during stage2 teardown. There is no
+	 * bound IPA address available here. Possibly dmabuf can help
+	 */
 	rme_unbind_vdev(realm, tdi->pdev, tdi->pdev->tsm->dsm_dev);
-
 	module_put(THIS_MODULE);
 }
 
Signature · 1 line
2.43.0
author maintainer
#147 lore raw

RFC PATCH 33/38 KVM: arm64: CCA: handle dev mem map/unmap

In reply to Aneesh Kumar K.V (Arm)

Handle VM exit on DEV_MEM_MAP

---
 arch/arm64/include/asm/rmi_cmds.h |  40 +++++++
 arch/arm64/include/asm/rmi_smc.h  |   5 +
 arch/arm64/kvm/rme-exit.c         |  39 +++++-
 arch/arm64/kvm/rme.c              | 190 ++++++++++++++++++++++++++++--
 drivers/vfio/pci/vfio_pci_core.c  |   1 +
 5 files changed, 262 insertions(+), 13 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +40 -0
index fcf6b319e953..900e35dae740 100644
@@ -638,4 +638,44 @@ static inline unsigned long rmi_vdev_complete(unsigned long rec_phys, unsigned l
 	return res.a0;
 }
 
+static inline int rmi_rtt_dev_mem_validate(unsigned long rd, unsigned long rec,
+					   unsigned long base, unsigned long top,
+					   unsigned long *out_top)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_RTT_DEV_MEM_VALIDATE, rd, rec, base, top, &res);
+
+	if (out_top)
+		*out_top = res.a1;
+
+	return res.a0;
+}
+
+static inline int rmi_dev_mem_map(unsigned long rd, unsigned long ipa,
+				  unsigned long level, unsigned long pa)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_DEV_MEM_MAP, rd, ipa, level, pa, &res);
+
+	return res.a0;
+}
+
+static inline int rmi_dev_mem_unmap(unsigned long rd, unsigned long ipa,
+				    unsigned long level, unsigned long *out_pa,
+				    unsigned long *out_ipa)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_DEV_MEM_UNMAP, rd, ipa, level, &res);
+
+	if (out_pa)
+		*out_pa = res.a1;
+	if (out_ipa)
+		*out_ipa = res.a2;
+
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +5 -0
index 7073eccaec5f..ab169b375198 100644
@@ -39,6 +39,7 @@
 
 #define SMC_RMI_RTT_READ_ENTRY		SMC_RMI_CALL(0x0161)
 #define SMC_RMI_RTT_UNMAP_UNPROTECTED	SMC_RMI_CALL(0x0162)
+#define SMC_RMI_RTT_DEV_MEM_VALIDATE	SMC_RMI_CALL(0x0163)
 
 #define SMC_RMI_PSCI_COMPLETE		SMC_RMI_CALL(0x0164)
 #define SMC_RMI_FEATURES		SMC_RMI_CALL(0x0165)
@@ -47,6 +48,9 @@
 #define SMC_RMI_RTT_INIT_RIPAS		SMC_RMI_CALL(0x0168)
 #define SMC_RMI_RTT_SET_RIPAS		SMC_RMI_CALL(0x0169)
 
+#define SMC_RMI_DEV_MEM_MAP		SMC_RMI_CALL(0x0172)
+#define SMC_RMI_DEV_MEM_UNMAP		SMC_RMI_CALL(0x0173)
+
 #define SMC_RMI_PDEV_COMMUNICATE        SMC_RMI_CALL(0x0175)
 #define SMC_RMI_PDEV_CREATE             SMC_RMI_CALL(0x0176)
 #define SMC_RMI_PDEV_DESTROY		SMC_RMI_CALL(0x0177)
@@ -84,6 +88,7 @@ enum rmi_ripas {
 	RMI_EMPTY = 0,
 	RMI_RAM = 1,
 	RMI_DESTROYED = 2,
+	RMI_DEV = 3,
 };
 
 #define RMI_NO_MEASURE_CONTENT	0
arch/arm64/kvm/rme-exit.c +33 -6
index 25948207fc5b..77829491805b 100644
@@ -170,17 +170,44 @@ EXPORT_SYMBOL_GPL(realm_exit_dev_mem_map_handler);
 static int rec_exit_dev_mem_map(struct kvm_vcpu *vcpu)
 {
 	int ret;
+	struct kvm *kvm = vcpu->kvm;
+	struct realm *realm = &kvm->arch.realm;
 	struct realm_rec *rec = &vcpu->arch.rec;
+	unsigned long base = rec->run->exit.dev_mem_base;
+	unsigned long top = rec->run->exit.dev_mem_top;
+
+	if (!kvm_realm_is_private_address(realm, base) ||
+	    !kvm_realm_is_private_address(realm, top - 1)) {
+		vcpu_err(vcpu, "Invalid DEV_MEM_VALIDATE for %#lx - %#lx\n", base, top);
+		return -EINVAL;
+	}
 
+	/* See if coco driver want to look at the dev mem_map request */
 	if (realm_exit_dev_mem_map_handler) {
 		ret = (*realm_exit_dev_mem_map_handler)(rec);
-	} else {
-		kvm_pr_unimpl("Unsupported exit reason: %u\n",
-			      rec->run->exit.exit_reason);
-		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
-		ret = 0;
+		if (ret)
+			return ret;
 	}
-	return ret;
+
+#if 0
+	/* we don't need a memory fault exit for device mapping.
+	 * 1. On enter to rec, we map the device memory using dev_mem_map
+	   2. There is no fallocate, and we are not tracking this via memory attributes.
+	   If we need a fault exit, we need to differentiate it in VMM so that we don't
+	   map the private memory via tsm map ioctl.
+	 */
+	/*
+	 * Exit to VMM so that VMM can deny the validation, the actual
+	 * validation response is done on next entry
+	 */
+	kvm_prepare_memory_fault_exit(vcpu, base, top - base, false, false,
+				      true);
+
+	/* exit to hypervisor */
+	return -EFAULT;
+#else
+	return 1;
+#endif
 }
 
 static void update_arch_timer_irq_lines(struct kvm_vcpu *vcpu)
arch/arm64/kvm/rme.c +183 -7

243 lines not on this page · raw message

drivers/vfio/pci/vfio_pci_core.c +1 -0
index afdb39c6aefd..264ee84d7ecd 100644
@@ -1718,6 +1718,7 @@ static const struct vm_operations_struct vfio_pci_mmap_ops = {
 #endif
 };
 
+/* FIXME!! don't allow mmap once the device is TDISP locked and we did dev mem_map. */
 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
 {
 	struct vfio_pci_core_device *vdev =
Signature · 1 line
2.43.0
author maintainer
#148 lore raw

RFC PATCH 34/38 coco: guest: arm64: Validate mmio range found in the interface report

In reply to Aneesh Kumar K.V (Arm)

This starts the sequence to transition the realm device to the TDISP RUN state by writing 1 to 'tsm/accept'.

---
 arch/arm64/include/asm/rsi_cmds.h         |  18 +++
 arch/arm64/include/asm/rsi_smc.h          |   4 +
 drivers/virt/coco/arm-cca-guest/arm-cca.c |   3 +
 drivers/virt/coco/arm-cca-guest/rsi-da.c  | 132 ++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h  |  25 ++++
 5 files changed, 182 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +18 -0
index 18fc4e1ce577..1cc00d404e53 100644
@@ -228,4 +228,22 @@ static inline unsigned long rsi_host_call(phys_addr_t addr)
 	return res.a0;
 }
 
+static inline unsigned long
+rsi_rdev_validate_mapping(unsigned long vdev_id, unsigned long inst_id,
+			  phys_addr_t start_ipa, phys_addr_t end_ipa,
+			  phys_addr_t io_pa, phys_addr_t *next_ipa, unsigned long flags)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_VALIDATE_MAPPING, vdev_id,
+			     inst_id, start_ipa, end_ipa, io_pa, flags, &res);
+	*next_ipa = res.a1;
+
+	if (res.a2 != RSI_ACCEPT)
+		return -EPERM;
+
+	if (res.a0 != RSI_SUCCESS)
+		return -EINVAL;
+	return 0;
+}
 #endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +4 -0
index 1d762fe3777b..a28b41cf01ca 100644
@@ -204,4 +204,8 @@ struct rsi_host_call {
 
 #define SMC_RSI_RDEV_LOCK			SMC_RSI_FID(0x1a9)
 
+#define RSI_DEV_MEM_COHERENT		BIT(0)
+#define RSI_DEV_MEM_LIMITED_ORDER	BIT(1)
+#define SMC_RSI_RDEV_VALIDATE_MAPPING		SMC_RSI_FID(0x1ac)
+
 #endif /* __ASM_RSI_SMC_H_ */
drivers/virt/coco/arm-cca-guest/arm-cca.c +3 -0
index de70fba09e92..c1cefb983ac7 100644
@@ -247,6 +247,9 @@ static void cca_tsm_unlock(struct pci_dev *pdev)
 	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
 		PCI_DEVID(pdev->bus->number, pdev->devfn);
 
+	/* invalidate dev mapping based on interface report */
+	rsi_update_interface_report(pdev, false);
+
 	ret = rhi_da_vdev_set_tdi_state(vdev_id, RHI_DA_TDI_CONFIG_UNLOCKED);
 	if (ret) {
 		pci_err(pdev, "failed to TSM unbind the device (%ld)\n", ret);
drivers/virt/coco/arm-cca-guest/rsi-da.c +132 -0

135 lines not on this page · raw message

drivers/virt/coco/arm-cca-guest/rsi-da.h +25 -0
index bd565785ff4b..0d6e1c0ada4a 100644
@@ -11,6 +11,28 @@
 #include <asm/rsi_smc.h>
 #include <asm/rhi.h>
 
+struct pci_tdisp_device_interface_report {
+	u16 interface_info;
+	u16 reserved;
+	u16 msi_x_message_control;
+	u16 lnr_control;
+	u32 tph_control;
+	u32 mmio_range_count;
+} __packed;
+
+struct pci_tdisp_mmio_range {
+	u64 first_page;
+	u32 num_pages;
+	u32 range_attributes;
+} __packed;
+
+#define TSM_INTF_REPORT_MMIO_MSIX_TABLE		BIT(0)
+#define TSM_INTF_REPORT_MMIO_PBA		BIT(1)
+#define TSM_INTF_REPORT_MMIO_IS_NON_TEE		BIT(2)
+#define TSM_INTF_REPORT_MMIO_IS_UPDATABLE	BIT(3)
+#define TSM_INTF_REPORT_MMIO_RESERVED		GENMASK(15, 4)
+#define TSM_INTF_REPORT_MMIO_RANGE_ID		GENMASK(31, 16)
+
 struct cca_guest_dsc {
 	struct pci_tsm_pf0 pci;
 	unsigned long instance_id;
@@ -29,5 +51,8 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
 	return container_of(tsm, struct cca_guest_dsc, pci.tsm);
 }
 
+int rsi_update_interface_report(struct pci_dev *pdev, bool validate);
 int rsi_device_lock(struct pci_dev *pdev);
+int rsi_device_start(struct pci_dev *pdev);
+
 #endif
Signature · 1 line
2.43.0
author maintainer
#161 lore raw

RFC PATCH 35/38 coco: guest: arm64: Add Realm device start and stop support

In reply to Aneesh Kumar K.V (Arm)

Writing 1 to 'tsm/acceept' will initiate the TDISP RUN sequence.

---
 arch/arm64/include/asm/rsi_cmds.h         | 19 +++++++
 arch/arm64/include/asm/rsi_smc.h          |  2 +
 drivers/virt/coco/arm-cca-guest/arm-cca.c | 15 ++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.c  | 60 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h  |  2 +-
 5 files changed, 97 insertions(+), 1 deletion(-)
arch/arm64/include/asm/rsi_cmds.h +19 -0
index 1cc00d404e53..3463d571d7db 100644
@@ -246,4 +246,23 @@ rsi_rdev_validate_mapping(unsigned long vdev_id, unsigned long inst_id,
 		return -EINVAL;
 	return 0;
 }
+
+static inline unsigned long __rsi_rdev_start(unsigned long vdev_id, unsigned long inst_id)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_START, vdev_id, inst_id, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long __rsi_rdev_stop(unsigned long vdev_id, unsigned long inst_id)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_STOP, vdev_id, inst_id, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +2 -0
index a28b41cf01ca..f6aa647239c0 100644
@@ -203,6 +203,8 @@ struct rsi_host_call {
 #define SMC_RSI_RDEV_GET_INTERFACE_REPORT	SMC_RSI_FID(0x1a6)
 
 #define SMC_RSI_RDEV_LOCK			SMC_RSI_FID(0x1a9)
+#define SMC_RSI_RDEV_START			SMC_RSI_FID(0x1aa)
+#define SMC_RSI_RDEV_STOP			SMC_RSI_FID(0x1ab)
 
 #define RSI_DEV_MEM_COHERENT		BIT(0)
 #define RSI_DEV_MEM_LIMITED_ORDER	BIT(1)
drivers/virt/coco/arm-cca-guest/arm-cca.c +15 -0
index c1cefb983ac7..7eeb9732d20a 100644
@@ -250,6 +250,8 @@ static void cca_tsm_unlock(struct pci_dev *pdev)
 	/* invalidate dev mapping based on interface report */
 	rsi_update_interface_report(pdev, false);
 
+	rsi_device_stop(pdev);
+
 	ret = rhi_da_vdev_set_tdi_state(vdev_id, RHI_DA_TDI_CONFIG_UNLOCKED);
 	if (ret) {
 		pci_err(pdev, "failed to TSM unbind the device (%ld)\n", ret);
@@ -257,11 +259,24 @@ static void cca_tsm_unlock(struct pci_dev *pdev)
 	}
 }
 
+static int cca_tsm_accept(struct pci_dev *pdev)
+{
+	int ret;
+
+	ret = rsi_device_start(pdev);
+	if (ret) {
+		pci_err(pdev, "failed to transition the device to run state (%d)\n", ret);
+		return -EIO;
+	}
+	return 0;
+}
+
 static const struct pci_tsm_ops cca_pci_ops = {
 	.probe = cca_tsm_pci_probe,
 	.remove = cca_tsm_pci_remove,
 	.lock = cca_tsm_lock,
 	.unlock = cca_tsm_unlock,
+	.accept	 = cca_tsm_accept,
 };
 
 static void cca_tsm_unregister(void *tsm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +60 -0

77 lines not on this page · raw message

drivers/virt/coco/arm-cca-guest/rsi-da.h +1 -1
index 0d6e1c0ada4a..71ee1edb832e 100644
@@ -54,5 +54,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
 int rsi_update_interface_report(struct pci_dev *pdev, bool validate);
 int rsi_device_lock(struct pci_dev *pdev);
 int rsi_device_start(struct pci_dev *pdev);
-
+int rsi_device_stop(struct pci_dev *pdev);
 #endif
Signature · 1 line
2.43.0
author maintainer
#164 lore raw

RFC PATCH 36/38 KVM: arm64: CCA: enable DA in realm create parameters

In reply to Aneesh Kumar K.V (Arm)

Now that we have all the required steps for DA in-place, enable DA while creating ralm.

---
 arch/arm64/include/asm/rmi_smc.h | 1 +
 arch/arm64/kvm/rme.c             | 2 ++
 2 files changed, 3 insertions(+)
arch/arm64/include/asm/rmi_smc.h +1 -0
index ab169b375198..f664954a2a91 100644
@@ -112,6 +112,7 @@ enum rmi_ripas {
 #define RMI_REALM_PARAM_FLAG_LPA2		BIT(0)
 #define RMI_REALM_PARAM_FLAG_SVE		BIT(1)
 #define RMI_REALM_PARAM_FLAG_PMU		BIT(2)
+#define RMI_REALM_PARAM_FLAG_DA			BIT(3)
 
 /*
  * Note many of these fields are smaller than u64 but all fields have u64
arch/arm64/kvm/rme.c +2 -0
index 11c8d47e3e9b..394d1534e6c2 100644
@@ -695,6 +695,8 @@ static int realm_create_rd(struct kvm *kvm)
 	if (r)
 		goto out_undelegate_tables;
 
+	/* For now default enable DA */
+	params->flags = RMI_REALM_PARAM_FLAG_DA;
 	params_phys = virt_to_phys(params);
 
 	if (rmi_realm_create(rd_phys, params_phys)) {
Signature · 1 line
2.43.0
author maintainer
#166 lore raw

RFC PATCH 37/38 coco: guest: arm64: Add support for fetching device measurements

In reply to Aneesh Kumar K.V (Arm)

Fetch device measurements using RSI_RDEV_GET_MEASUREMENTS.

---
 arch/arm64/include/asm/rsi_cmds.h        | 11 +++++++
 arch/arm64/include/asm/rsi_smc.h         | 16 ++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.c | 39 ++++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h |  2 ++
 4 files changed, 68 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +11 -0
index 3463d571d7db..42b998f44a0e 100644
@@ -265,4 +265,15 @@ static inline unsigned long __rsi_rdev_stop(unsigned long vdev_id, unsigned long
 	return res.a0;
 }
 
+static inline unsigned long __rsi_rdev_get_measurements(unsigned long vdev_id,
+						       unsigned long inst_id,
+						       phys_addr_t meas)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_GET_MEASUREMENTS, vdev_id, inst_id, meas, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +16 -0
index f6aa647239c0..f051db54cdc3 100644
@@ -202,6 +202,22 @@ struct rsi_host_call {
 
 #define SMC_RSI_RDEV_GET_INTERFACE_REPORT	SMC_RSI_FID(0x1a6)
 
+#define RSI_DEV_MEASURE_ALL		BIT(0)
+#define RSI_DEV_MEASURE_SIGNED		BIT(1)
+#define RSI_DEV_MEASURE_RAW		BIT(2)
+
+struct rsi_device_measurements_params {
+	union {
+		struct {
+			u64 flags;
+			u8 indices[32];
+			u8 nounce[32];
+		};
+		u8 padding[0x100];
+	};
+};
+
+#define SMC_RSI_RDEV_GET_MEASUREMENTS		SMC_RSI_FID(0x1a7)
 #define SMC_RSI_RDEV_LOCK			SMC_RSI_FID(0x1a9)
 #define SMC_RSI_RDEV_START			SMC_RSI_FID(0x1aa)
 #define SMC_RSI_RDEV_STOP			SMC_RSI_FID(0x1ab)
drivers/virt/coco/arm-cca-guest/rsi-da.c +39 -0
index 64034d220e02..6222b10964ee 100644
@@ -166,10 +166,31 @@ static long rhi_get_report(int vdev_id, int da_object_type, void **report, int *
 	return ret;
 }
 
+static inline unsigned long
+rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id,
+			  unsigned long inst_id, phys_addr_t meas)
+{
+	unsigned long ret;
+
+	ret = __rsi_rdev_get_measurements(vdev_id, inst_id, meas);
+	if (ret != RSI_SUCCESS)
+		return ret;
+
+	do {
+		ret = rsi_rdev_continue(vdev_id, inst_id);
+	} while (ret == RSI_INCOMPLETE);
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to communicate with the device (%lu)\n", ret);
+		return ret;
+	}
+	return RSI_SUCCESS;
+}
+
 int rsi_device_lock(struct pci_dev *pdev)
 {
 	unsigned long ret;
 	unsigned long tdisp_version;
+	struct rsi_device_measurements_params *rsi_dev_meas;
 	struct cca_guest_dsc *dsm = to_cca_guest_dsc(pdev);
 	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
 		PCI_DEVID(pdev->bus->number, pdev->devfn);
@@ -198,6 +219,17 @@ int rsi_device_lock(struct pci_dev *pdev)
 		return -EOPNOTSUPP;
 	}
 
+	rsi_dev_meas = (struct rsi_device_measurements_params *)__get_free_page(GFP_KERNEL);
+	rsi_dev_meas->flags = RSI_DEV_MEASURE_ALL;
+	ret = rsi_rdev_get_measurements(pdev, vdev_id, dsm->instance_id,
+					virt_to_phys(rsi_dev_meas));
+
+	free_page((unsigned long)rsi_dev_meas);
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to get device measurement (%lu)\n", ret);
+		return -EIO;
+	}
+
 	/* Now make a host call to copy the interface report to guest. */
 	ret = rhi_get_report(vdev_id, RHI_DA_OBJECT_INTERFACE_REPORT,
 			     &dsm->interface_report, &dsm->interface_report_size);
@@ -213,6 +245,13 @@ int rsi_device_lock(struct pci_dev *pdev)
 		return -EIO;
 	}
 
+	ret = rhi_get_report(vdev_id, RHI_DA_OBJECT_MEASUREMENT,
+			     &dsm->measurements, &dsm->measurements_size);
+	if (ret) {
+		pci_err(pdev, "failed to get device certificate from the host (%lu)\n", ret);
+		return -EIO;
+	}
+
 	return ret;
 }
 static inline unsigned long rsi_rdev_start(struct pci_dev *pdev,
drivers/virt/coco/arm-cca-guest/rsi-da.h +2 -0
index 71ee1edb832e..f26156d9be81 100644
@@ -40,6 +40,8 @@ struct cca_guest_dsc {
 	int interface_report_size;
 	void *certificate;
 	int certificate_size;
+	void *measurements;
+	int measurements_size;
 };
 
 static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
Signature · 1 line
2.43.0
author maintainer
#169 lore raw

RFC PATCH 38/38 coco: guest: arm64: Add support for fetching device info

In reply to Aneesh Kumar K.V (Arm)

RSI_RDEV_GET_INFO returns different digest hash values, which can be compared with host cached values to ensure the host didn't tamper with the cached data.

---
 arch/arm64/include/asm/rsi_cmds.h        |  12 +++
 arch/arm64/include/asm/rsi_smc.h         |  24 +++++
 drivers/virt/coco/arm-cca-guest/Kconfig  |   2 +
 drivers/virt/coco/arm-cca-guest/rsi-da.c | 128 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h |  13 +++
 5 files changed, 179 insertions(+)
arch/arm64/include/asm/rsi_cmds.h +12 -0
index 42b998f44a0e..6b90a6cdd7fb 100644
@@ -276,4 +276,16 @@ static inline unsigned long __rsi_rdev_get_measurements(unsigned long vdev_id,
 	return res.a0;
 }
 
+static inline unsigned long rsi_rdev_get_info(unsigned long vdev_id,
+					      unsigned long inst_id,
+					      unsigned long digest_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RSI_RDEV_GET_INFO,
+			     vdev_id, inst_id, digest_phys, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RSI_CMDS_H */
arch/arm64/include/asm/rsi_smc.h +24 -0
index f051db54cdc3..e2b51cf58bd4 100644
@@ -125,6 +125,9 @@
 
 #ifndef __ASSEMBLY__
 
+#define RSI_HASH_SHA_256 0
+#define RSI_HASH_SHA_512 1
+
 struct realm_config {
 	union {
 		struct {
@@ -200,6 +203,27 @@ struct rsi_host_call {
 #define SMC_RSI_RDEV_GET_INSTANCE_ID		SMC_RSI_FID(0x19c)
 #define SMC_RSI_RDEV_CONTINUE			SMC_RSI_FID(0x1a4)
 
+struct rsi_device_info {
+	union {
+		struct {
+			u64 flags;
+			u64 attest_type;
+			u64 cert_id;
+			u8 hash_algo;
+		};
+		u8 padding[0x40];
+	};
+	union { /* 0x40  */
+		struct {
+			u8 cert_digest[0x40];
+			u8 meas_digest[0x40];
+			u8 report_digest[0x40];
+		};
+		u8 padding2[0x200 - 0x40];
+	};
+};
+
+#define SMC_RSI_RDEV_GET_INFO			SMC_RSI_FID(0x1a5)
 #define SMC_RSI_RDEV_GET_INTERFACE_REPORT	SMC_RSI_FID(0x1a6)
 
 #define RSI_DEV_MEASURE_ALL		BIT(0)
drivers/virt/coco/arm-cca-guest/Kconfig +2 -0
index 410d9c3fb2b3..6fc86c1f3900 100644
@@ -5,6 +5,8 @@ config ARM_CCA_GUEST
 	tristate "Arm CCA Guest driver"
 	depends on ARM64
 	depends on PCI_TSM
+	select CRYPTO_SHA256
+	select CRYPTO_SHA512
 	select TSM_REPORTS
 	select TSM
 	help
drivers/virt/coco/arm-cca-guest/rsi-da.c +128 -0

153 lines not on this page · raw message

drivers/virt/coco/arm-cca-guest/rsi-da.h +13 -0
index f26156d9be81..e8953b8e85a3 100644
@@ -10,6 +10,7 @@
 #include <linux/pci-tsm.h>
 #include <asm/rsi_smc.h>
 #include <asm/rhi.h>
+#include <crypto/sha2.h>
 
 struct pci_tdisp_device_interface_report {
 	u16 interface_info;
@@ -33,6 +34,17 @@ struct pci_tdisp_mmio_range {
 #define TSM_INTF_REPORT_MMIO_RESERVED		GENMASK(15, 4)
 #define TSM_INTF_REPORT_MMIO_RANGE_ID		GENMASK(31, 16)
 
+struct dsm_device_info {
+	u64 flags;
+	u64 attest_type;
+	u64 cert_id;
+	u64 hash_algo;
+	u8 cert_digest[SHA512_DIGEST_SIZE];
+	u8 meas_digest[SHA512_DIGEST_SIZE];
+	u8 report_digest[SHA512_DIGEST_SIZE];
+};
+
+
 struct cca_guest_dsc {
 	struct pci_tsm_pf0 pci;
 	unsigned long instance_id;
@@ -42,6 +54,7 @@ struct cca_guest_dsc {
 	int certificate_size;
 	void *measurements;
 	int measurements_size;
+	struct dsm_device_info dev_info;
 };
 
 static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
Signature · 1 line
2.43.0
#39 lore raw

Re: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance

In reply to Aneesh Kumar K.V (Arm)
Quoted 18 lines · Aneesh Kumar K.V (Arm)
drivers/iommu/iommufd/viommu.c +18 -0
+#if IS_ENABLED(CONFIG_KVM)
+#include <linux/kvm_host.h>
+
+static int viommu_get_kvm(struct iommufd_viommu *viommu, int kvm_vm_fd)
+{
+	int rc = -EBADF;
+	struct file *filp;
+
+	filp = fget(kvm_vm_fd);
+
+	if (!file_is_kvm(filp))
+		goto err_out;
+
+	/* hold the kvm reference via file descriptor */
+	viommu->kvm_filp = filp;
+	return 0;
+err_out:
+	viommu->kvm_filp = NULL;

drivers/iommu/iommufd/viommu.c line 34

Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10 -0
+	fput(filp);
+	return rc;
+}
+
+static void viommu_put_kvm(struct iommufd_viommu *viommu)
+{
+	fput(viommu->kvm_filp);
+	viommu->kvm_filp = NULL;
+}
+#endif

Missing stub functions for !CONFIG_KVM?

Looks like an OK design otherwise

include/uapi/linux/iommufd.h struct iommu_viommu_alloc { · line 1071

Quoted 5 lines
include/uapi/linux/iommufd.h +1 -0
@@ -1057,6 +1068,7 @@ struct iommu_viommu_alloc {
 	__u32 data_len;
 	__u32 __reserved;
 	__aligned_u64 data_uptr;
+	__u32 kvm_vm_fd;

fds are __s32, they are signed numbers.

Jason

#50 lore raw

Re: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl

In reply to Aneesh Kumar K.V (Arm)
Quoted 5 lines · Aneesh Kumar K.V (Arm)

With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent

I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?

But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??

I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.

Jason

#7 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Aneesh Kumar K.V (Arm)

arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56

Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5 -0
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
 	return crypt_ops->decrypt(addr, numpages);
 }
 EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;

Is this OK? I see code like this:

static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)

{
	if (force_dma_unencrypted(dev))
		return phys_to_dma_unencrypted(dev, phys);
	return phys_to_dma(dev, phys);

What are the ARM rules for generating dma addreses?

1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"

2) Device is T=1, memory is encrypted, use the phys_to_dma() normally

3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?

Can you document this in a comment?

include/linux/device.h struct device { · line 692

Quoted 10 lines
include/linux/device.h +1 -0
index 4940db137fff..d62e0dd9d8ee 100644
@@ -688,6 +688,7 @@ struct device {
 #ifdef CONFIG_IOMMU_DMA
 	bool			dma_iommu:1;
 #endif
+	bool			tdi_enabled:1;
 };

I would give the dev->tdi_enabled a clearer name, maybe dev->encrypted_dma_supported ?

Also need to think carefully of a bitfield is OK here, we can't locklessly change a bitfield so need to audit that all members are set under, probably, the device lock or some other single threaded hand waving. It seems believable it is like that but should be checked out, and add a lockdep if it relies on the device lock.

Jason

author maintainer
#8 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

Quoted 25 lines · Jason Gunthorpe

arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56

Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5 -0
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
 	return crypt_ops->decrypt(addr, numpages);
 }
 EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;

Is this OK? I see code like this:

static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)

{
	if (force_dma_unencrypted(dev))
		return phys_to_dma_unencrypted(dev, phys);
	return phys_to_dma(dev, phys);

What are the ARM rules for generating dma addreses?

1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"

Quoted 8 lines

2) Device is T=1, memory is encrypted, use the phys_to_dma() normally

3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?

Can you document this in a comment?

If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported. It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

Based on this assumption, T=1 devices will always access private/encrypted memory, while T=0 devices will be restricted to shared/unencrypted memory.

Quoted 15 lines

include/linux/device.h struct device { · line 692

Quoted 11 lines
include/linux/device.h +1 -0
index 4940db137fff..d62e0dd9d8ee 100644
@@ -688,6 +688,7 @@ struct device {
 #ifdef CONFIG_IOMMU_DMA
 	bool			dma_iommu:1;
 #endif
+	bool			tdi_enabled:1;
 };

I would give the dev->tdi_enabled a clearer name, maybe dev->encrypted_dma_supported ?

Quoted 6 lines

Also need to think carefully of a bitfield is OK here, we can't locklessly change a bitfield so need to audit that all members are set under, probably, the device lock or some other single threaded hand waving. It seems believable it is like that but should be checked out, and add a lockdep if it relies on the device lock.

Will check and update the patch.

-aneesh

author maintainer
#30 lore raw

Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE

In reply to Jason Gunthorpe
Quoted 11 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)

Why would we need this?

I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?

But also why is it an iommufd option? That doesn't seem right..

Jason

This is based on our previous discussion https://lore.kernel.org/all/[email protected]

IIUC, we intend to request the resource in exclusive mode for secure guests—regardless of whether the platform is Intel or ARM. Could you help clarify the MCE issue observed on Intel platforms in this context?

-aneesh

author maintainer
#40 lore raw

Re: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance

In reply to Jason Gunthorpe
Quoted 35 lines · Jason Gunthorpe
Quoted 18 lines · Aneesh Kumar K.V (Arm)
drivers/iommu/iommufd/viommu.c +18 -0
+#if IS_ENABLED(CONFIG_KVM)
+#include <linux/kvm_host.h>
+
+static int viommu_get_kvm(struct iommufd_viommu *viommu, int kvm_vm_fd)
+{
+	int rc = -EBADF;
+	struct file *filp;
+
+	filp = fget(kvm_vm_fd);
+
+	if (!file_is_kvm(filp))
+		goto err_out;
+
+	/* hold the kvm reference via file descriptor */
+	viommu->kvm_filp = filp;
+	return 0;
+err_out:
+	viommu->kvm_filp = NULL;

drivers/iommu/iommufd/viommu.c line 34

Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10 -0
+	fput(filp);
+	return rc;
+}
+
+static void viommu_put_kvm(struct iommufd_viommu *viommu)
+{
+	fput(viommu->kvm_filp);
+	viommu->kvm_filp = NULL;
+}
+#endif

Missing stub functions for !CONFIG_KVM?

Looks like an OK design otherwise

Quoted 8 lines

include/uapi/linux/iommufd.h struct iommu_viommu_alloc { · line 1071

Quoted 5 lines
include/uapi/linux/iommufd.h +1 -0
@@ -1057,6 +1068,7 @@ struct iommu_viommu_alloc {
 	__u32 data_len;
 	__u32 __reserved;
 	__aligned_u64 data_uptr;
+	__u32 kvm_vm_fd;

fds are __s32, they are signed numbers.

Thanks for the review comments. I’ll update the patch with the suggested changes.

-aneesh

author maintainer
#51 lore raw

Re: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl

In reply to Jason Gunthorpe
Quoted 10 lines · Jason Gunthorpe
Quoted 5 lines · Aneesh Kumar K.V (Arm)

With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent

I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?

That is correct.

Quoted 5 lines

But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??

This functionality is equivalent to IOMMU_IOAS_MAP, but in the presence of firmware like RMM, we also need to supply the realm descriptor associated with the KVM instance.

Initially, I attempted to handle this within the map_pages callback in iommu_domain_ops, but that path lacks any awareness of the associated KVM context, making it unsuitable for this purpose.

Quoted 5 lines

I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.

We need to allocate and free these pages dynamically as they are converted between private and shared states.

-aneesh

#31 lore raw

Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE

In reply to Aneesh Kumar K.V
Quoted 7 lines · Aneesh Kumar K.V
Quoted 5 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)

Why would we need this?

Quoted 8 lines
Quoted 6 lines

I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?

But also why is it an iommufd option? That doesn't seem right..

Jason

This is based on our previous discussion https://lore.kernel.org/all/[email protected]

I suggested a global option, this is a per-device option, and that especially seems wrong for iommufd. If it is per-device that is vfio, if it is global then vfio can pick it up during the early phases of opening the device.

Quoted 3 lines

IIUC, we intend to request the resource in exclusive mode for secure guests—regardless of whether the platform is Intel or ARM. Could you help clarify the MCE issue observed on Intel platforms in this context?

As I understand it Intel MCEs if the non-secure side ever reads from secure'd address space. So there is alot of emphasis there to ensure there are no CPU mappings.

Jason

#52 lore raw

Re: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl

In reply to Aneesh Kumar K.V
Quoted 7 lines · Aneesh Kumar K.V
Quoted 4 lines

But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??

This functionality is equivalent to IOMMU_IOAS_MAP, but in the presence of firmware like RMM, we also need to supply the realm descriptor associated with the KVM instance.

There is no IOAS here because the secure world is using the KVM page table. Since KVM owns this I don't see why iommufd should be invovled in any way.

You need KVM to push the guestmemfd to the RMM and pin all the memory.

Quoted 7 lines
Quoted 4 lines

I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.

We need to allocate and free these pages dynamically as they are converted between private and shared states.

That's still within guestmemfd's area of concern and it can immediately pin on state changes.

Jason

#9 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Aneesh Kumar K.V
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe

arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56

Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5 -0
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
 	return crypt_ops->decrypt(addr, numpages);
 }
 EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;

Is this OK? I see code like this:

static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)

{
	if (force_dma_unencrypted(dev))
		return phys_to_dma_unencrypted(dev, phys);
	return phys_to_dma(dev, phys);

What are the ARM rules for generating dma addreses?

1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"

2) Device is T=1, memory is encrypted, use the phys_to_dma() normally

3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?

Quoted 5 lines
Quoted 2 lines

Can you document this in a comment?

If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.

No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.

Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

PCI SIG supports it, kernel should support it.

Jason

#44 lore raw

Re: [RFC PATCH v1 08/38] iommufd/tsm: Add tsm_op iommufd ioctls

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:45 +0530 "Aneesh Kumar K.V (Arm)"…

Add operations bind and unbind used to bind a TDI to the secure guest.

Hi Aneesh,

I'm mostly reading this to get head around it rather than fully review at this point.

A few things inline though that I noticed whilst doing so.

Jonathan

Quoted 26 lines
---
 drivers/iommu/iommufd/iommufd_private.h |  1 +
 drivers/iommu/iommufd/main.c            |  3 ++
 drivers/iommu/iommufd/viommu.c          | 50 +++++++++++++++++++++++++
 drivers/vfio/pci/vfio_pci_core.c        | 10 +++++
 include/uapi/linux/iommufd.h            | 18 +++++++++
 5 files changed, 82 insertions(+)
drivers/iommu/iommufd/iommufd_private.h +1 -0
index fce68714c80f..e08186f1d102 100644
@@ -697,6 +697,7 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj);
 void iommufd_vdevice_abort(struct iommufd_object *obj);
 int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
 void iommufd_hw_queue_destroy(struct iommufd_object *obj);
+int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd);
 
 static inline struct iommufd_vdevice *
 iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id)
drivers/iommu/iommufd/viommu.c +0 -0
index 59f1e1176f7f..c934312e5397 100644

drivers/iommu/iommufd/viommu.c void iommufd_vdevice_abort(struct iommufd_object *obj) · line 165

Quoted 4 lines
drivers/iommu/iommufd/viommu.c +1 -0
@@ -162,6 +162,9 @@ void iommufd_vdevice_abort(struct iommufd_object *obj)
 
 	lockdep_assert_held(&idev->igroup->lock);
 
+#ifdef CONFIG_TSM

Can we use stubs for some of this stuff so we don't need ifdefs in as many places.

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 479

Quoted 12 lines
drivers/iommu/iommufd/viommu.c +5 -0
+	tsm_unbind(idev->dev);
+#endif
 	if (vdev->destroy)
 		vdev->destroy(vdev);
 	/* xa_cmpxchg is okay to fail if alloc failed xa_cmpxchg previously */
@@ -471,3 +474,50 @@ int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	iommufd_put_object(ucmd->ictx, &viommu->obj);
 	return rc;
 }
+
+#ifdef CONFIG_TSM
+int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd)

Might want to split this out to a separate c file and use stubs in a header to keep the code clean here.

Quoted 9 lines
drivers/iommu/iommufd/viommu.c +9 -0
+{
+	struct iommu_vdevice_tsm_op *cmd = ucmd->cmd;
+	struct iommufd_vdevice *vdev;
+	struct kvm *kvm;
+	int rc = -ENODEV;
+
+	if (cmd->flags)
+		return -EOPNOTSUPP;
+

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 498

Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10 -0
+	vdev = container_of(iommufd_get_object(ucmd->ictx, cmd->vdevice_id,
+					       IOMMUFD_OBJ_VDEVICE),
+			    struct iommufd_vdevice, obj);
+	if (IS_ERR(vdev))
+		return PTR_ERR(vdev);
+
+	kvm = vdev->viommu->kvm_filp->private_data;
+	if (kvm) {
+		/*
+		 * tsm layer will make take care of parallel calls to tsm_bind/unbind

Wrap comment to say under 80 chars. Or if file goes higher, use a single line comment.

tsm layer will take care ...

(stray 'make')

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 506

Quoted 8 lines
drivers/iommu/iommufd/viommu.c +8 -0
+		 */
+		if (cmd->op == IOMMU_VDEICE_TSM_BIND)
+			rc = tsm_bind(vdev->idev->dev, kvm, vdev->virt_id);
+		else if (cmd->op == IOMMU_VDEICE_TSM_UNBIND)
+			rc = tsm_unbind(vdev->idev->dev);
+
+		if (rc) {
+			rc = -ENODEV;

If we want to eat an error code coming from elsewhere, maybe a comment on why?

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 510

Quoted 4 lines
drivers/iommu/iommufd/viommu.c +4 -0
+			goto out_put_vdev;
+		}
+	} else {
+		goto out_put_vdev;

If this always skips the next line, does that imply that line should have been under if (kvm)? Maybe this makes more sense in later patches - if so ignore this comment.

Quoted 18 lines
drivers/iommu/iommufd/viommu.c +13 -0
+	}
+	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
+
+out_put_vdev:
+	iommufd_put_object(ucmd->ictx, &vdev->obj);
+	return rc;
+}
+#else /* !CONFIG_TSM */
+int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd)
+{
+	return -ENODEV;
+}
+#endif
drivers/vfio/pci/vfio_pci_core.c +0 -0
index bee3cf3226e9..afdb39c6aefd 100644

drivers/vfio/pci/vfio_pci_core.c void vfio_pci_core_close_device(struct vfio_device *core_vdev) · line 698

Quoted 5 lines
drivers/vfio/pci/vfio_pci_core.c +2 -0
@@ -694,6 +694,16 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
 #if IS_ENABLED(CONFIG_EEH)
 	eeh_dev_release(vdev->pdev);
 #endif
+
+#if 0

If you really need to do this, add a comment on why if 0

Quoted 19 lines
drivers/vfio/pci/vfio_pci_core.c +8 -0
+	/*
+	 * destroy vdevice which involves tsm unbind before we disable pci disable
+	 * A MSE/BME clear will transition the device to error state.
+	 */
+	if (core_vdev->iommufd_device)
+		iommufd_device_tombstone_vdevice(core_vdev->iommufd_device);
+#endif
+
 	vfio_pci_core_disable(vdev);
 
 	mutex_lock(&vdev->igate);
include/uapi/linux/iommufd.h +0 -0
index 9014c61a97d4..8b1fbf1ef25c 100644
@@ -57,6 +57,7 @@ enum {
 
 /**

include/uapi/linux/iommufd.h enum iommu_veventq_flag { · line 1133

Quoted 6 lines
include/uapi/linux/iommufd.h +3 -0
@@ -1127,6 +1128,23 @@ enum iommu_veventq_flag {
 	IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0),
 };
 
+/**
+ * struct iommu_vdevice_tsm_OP - ioctl(IOMMU_VDEVICE_TSM_OP)
+ * @size: sizeof(struct iommu_vdevice_tsm_OP)

_op I guess?

Quoted 17 lines
include/uapi/linux/iommufd.h +14 -0
+ * @op: Either TSM_BIND or TSM_UNBIMD
+ * @flags: Must be 0
+ * @vdevice_id: Object handle for the vDevice. Returned from IOMMU_VDEVICE_ALLOC
+ */
+struct iommu_vdevice_tsm_op {
+	__u32 size;
+	__u32 op;
+	__u32 flags;
+	__u32 vdevice_id;
+};
+#define IOMMU_VDEVICE_TSM_OP	_IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_OP)
+#define IOMMU_VDEICE_TSM_BIND		0x1
+#define IOMMU_VDEICE_TSM_UNBIND		0x2
+
 /**
  * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status
  * @flags: Combination of enum iommu_veventq_flag
#41 lore raw

Re: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance

In reply to Aneesh Kumar K.V (Arm)
Quoted 28 lines · On Mon, 28 Jul 2025 19:21:44 +0530 "Aneesh Kumar K.V (Arm)"…

The associated kvm instance will be used in later patch by iommufd to bind a tdi to kvm.

---
 drivers/iommu/iommufd/viommu.c | 45 +++++++++++++++++++++++++++++++++-
 include/linux/iommufd.h        |  3 +++
 include/uapi/linux/iommufd.h   | 12 +++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)
drivers/iommu/iommufd/viommu.c +10 -0
index 2ca5809b238b..59f1e1176f7f 100644
@@ -2,6 +2,36 @@
 /* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
  */
 #include "iommufd_private.h"
+#include "linux/tsm.h"
+
+#if IS_ENABLED(CONFIG_KVM)
+#include <linux/kvm_host.h>
+
+static int viommu_get_kvm(struct iommufd_viommu *viommu, int kvm_vm_fd)
+{
+	int rc = -EBADF;
+	struct file *filp;
+

drivers/iommu/iommufd/viommu.c line 24

Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10 -0
+	filp = fget(kvm_vm_fd);
+
+	if (!file_is_kvm(filp))
+		goto err_out;
+
+	/* hold the kvm reference via file descriptor */
+	viommu->kvm_filp = filp;
+	return 0;
+err_out:
+	viommu->kvm_filp = NULL;

Is this to undo side effects from this function on error?

kvm_filp is only set after all error paths so maybe this isn't needed?

If this isn't needed then use __free(fput) and no_free_ptr() to deal with filp more simply and in teh erorr path can just return -EBADF directly rather than the goto.

Or are we avoiding that stuff in iommufd?

Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10 -0
+	fput(filp);
+	return rc;
+}
+
+static void viommu_put_kvm(struct iommufd_viommu *viommu)
+{
+	fput(viommu->kvm_filp);
+	viommu->kvm_filp = NULL;
+}
+#endif
#58 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Aneesh Kumar K.V (Arm)
Quoted 4 lines · On Mon, 28 Jul 2025 19:21:48 +0530 "Aneesh Kumar K.V (Arm)"…

Register a platform device if the CCA DA feature is supported. A driver for this platform device will further drive the CCA DA workflow.

Few trivial things

Quoted 13 lines
arch/arm64/kvm/rme.c +1 -0
index ec8093ec2da3..d1c147aba2ed 100644
@@ -4,6 +4,7 @@
  */
 
 #include <linux/kvm_host.h>
+#include <linux/platform_device.h>
 
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_mmu.h>

arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1728

Quoted 4 lines
arch/arm64/kvm/rme.c +1 -0
@@ -1724,6 +1725,18 @@ int kvm_init_realm_vm(struct kvm *kvm)
 	return 0;
 }
 
+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1730

Quoted 2 lines
arch/arm64/kvm/rme.c +2 -0
+	.name = RMI_DEV_NAME,
+	.id = PLATFORM_DEVID_NONE

Add trailing comma. More than possible something else will be added after this.

arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1736

Quoted 6 lines
arch/arm64/kvm/rme.c +6 -0
+};
+
+static void rmm_tsm_init(void)
+{
+	if (!platform_device_register(&cca_host_dev))
+		pr_info("CCA host DA platform device initialized.\n");

Noisy as we should be able to tell that in a bunch of other ways.

arch/arm64/kvm/rme.c int kvm_init_realm_vm(struct kvm *kvm) · line 1737

Quoted 1 line
arch/arm64/kvm/rme.c +1 -0
+

Excess blank line.

Quoted 1 line
arch/arm64/kvm/rme.c +1 -0
+}
#91 lore raw

Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:49 +0530 "Aneesh Kumar K.V (Arm)"…

This driver registers the pci_tsm_ops with tsm subsystem.

Hi Aneesh,

For this main comment is around use of __free.. Dan wrote up guidance and added to cleanup.h after many email threads kept running into same issues and Linus added his requirements for that stuff to be acceptable.

Anyhow, easy to fix - comments inline.

Quoted 8 lines
drivers/virt/coco/arm-cca-host/Kconfig new file +2 -0
new file mode 100644
index 000000000000..0f19fbf47613
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#

drivers/virt/coco/arm-cca-host/Kconfig line 12

Quoted 10 lines
drivers/virt/coco/arm-cca-host/Kconfig new file +10 -0
+# TSM (TEE Security Manager) host drivers
+#
+config ARM_CCA_HOST
+	tristate "Arm CCA Host driver"
+	depends on ARM64
+	depends on PCI_TSM
+	select TSM
+
+	help
+	  The driver provides TSM backend for ARM CCA

That's going to make for grumpy checkpatch! More help.

Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +2 -0
new file mode 100644
index 000000000000..c8b0e6db1f47
@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*

drivers/virt/coco/arm-cca-host/arm-cca.c line 12

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10 -0
+ * Copyright (C) 2025 ARM Ltd.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/pci-tsm.h>
+#include <linux/pci-ide.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/tsm.h>
+#include <linux/vmalloc.h>

cleanup.h and maybe others missing. Basically follow include what you use principles (flexed a little for headers that are front ends to others).

Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5 -0
+
+#include "rmm-da.h"
+
+/* Number of streams that we can support at the hostbridge level */
+#define CCA_HB_PLATFORM_STREAMS 4

drivers/virt/coco/arm-cca-host/arm-cca.c line 27

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10 -0
+
+/* Total number of stream id supported at root port level */
+#define MAX_STREAM_ID	256
+
+DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T))
+static struct pci_tsm *cca_tsm_pci_probe(struct pci_dev *pdev)
+{
+	int rc;
+	struct pci_host_bridge *hb;
+	struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = NULL;

Read the stuff in cleanup.h and work out why this needs changing to be inline below and not use this NULL pattern here (unless you like grumpy Linus ;)

Note that with the err_out, even if you do that you'll still be breaking with the guidance doc (and actually causing undefined behavior :) Get rid of those gotos if you want to use __free()

Quoted 24 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +24 -0
+
+	if (pdev->is_virtfn)
+		return NULL;
+
+	if (!is_pci_tsm_pf0(pdev)) {
+		struct pci_tsm *tsm = kzalloc(sizeof(*tsm), GFP_KERNEL);
+
+		if (!tsm)
+			goto err_out;
+
+		pci_tsm_initialize(pdev, tsm);
+		return tsm;
+	}
+
+	if (!pdev->ide_cap)
+		goto err_out;
+
+	dsc_pf0 = vcalloc(sizeof(*dsc_pf0), GFP_KERNEL);
+	if (!dsc_pf0)
+		goto err_out;
+
+	rc = pci_tsm_pf0_initialize(pdev, &dsc_pf0->pci);
+	if (rc)
+		return NULL;

drivers/virt/coco/arm-cca-host/arm-cca.c line 61

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10 -0
+	/*
+	 * FIXME!!
+	 * update the hostbridge details. This should go into
+	 * some host bridge probe/init routine.
+	 * than the selective index supported by the endpoint
+	 */
+	hb = pci_find_host_bridge(pdev->bus);
+	pci_ide_init_nr_streams(hb, CCA_HB_PLATFORM_STREAMS);
+
+	pci_info(pdev, "tsm enabled\n");

Ok. RFC I guess. Still pci_dbg()

drivers/virt/coco/arm-cca-host/arm-cca.c line 64

Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +3 -0
+	return &no_free_ptr(dsc_pf0)->pci.tsm;
+
+err_out:

Why? Random mix of direct returns of NULL above and goto here.

Quoted 36 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +35 -0
+	return NULL;
+}
+
+static void cca_tsm_pci_remove(struct pci_tsm *tsm)
+{
+	struct pci_dev *pdev = tsm->pdev;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+
+	if (WARN_ON(pdev->is_virtfn))
+		return;
+
+	if (!is_pci_tsm_pf0(pdev)) {
+
+		pci_dbg(tsm->pdev, "tsm disabled\n");
+		kfree(pdev->tsm);
+		return;
+	}
+
+	dsc_pf0 = to_cca_dsc_pf0(pdev);
+	pci_dbg(tsm->pdev, "tsm disabled\n");
+	vfree(dsc_pf0);
+}
+
+/* per root port unique with multiple restrictions. For now global */
+static DECLARE_BITMAP(cca_stream_ids, MAX_STREAM_ID);
 
+
+static void cca_tsm_disconnect(struct pci_dev *pdev)
+{
+	struct pci_dev *rp = pcie_find_root_port(pdev);
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct pci_ide *ide;
+
+	if (WARN_ON(!is_pci_tsm_pf0(pdev)))
+		return;
+

drivers/virt/coco/arm-cca-host/arm-cca.c line 165

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10 -0
+	dsc_pf0 = to_cca_dsc_pf0(pdev);
+	ide = dsc_pf0->sel_stream;
+	dsc_pf0->sel_stream = NULL;
+	pci_ide_stream_disable(pdev, ide);
+	tsm_ide_stream_unregister(ide);
+	pci_ide_stream_teardown(rp, ide);
+	pci_ide_stream_teardown(pdev, ide);
+	pci_ide_stream_unregister(ide);
+	clear_bit(ide->stream_id, cca_stream_ids);
+	pci_ide_stream_free(ide);

Ordering subtly different from error path above.
If there is a reason for that add a comment.

Quoted 6 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5 -0
+}
 
+static void cca_tsm_remove(void *tsm_core)
+{
+	tsm_unregister(tsm_core);
+}

drivers/virt/coco/arm-cca-host/arm-cca.c line 188

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10 -0
+
+static int cca_tsm_probe(struct platform_device *pdev)
+{
+	struct tsm_core_dev *tsm_core;
+
+	tsm_core = tsm_register(&pdev->dev, NULL, &cca_pci_ops);
+	if (IS_ERR(tsm_core))
+		return PTR_ERR(tsm_core);
+
+	return devm_add_action_or_reset(&pdev->dev, cca_tsm_remove, tsm_core);

So this makes two with the one in Dan's test code. devm_tsm_register() seems to be a useful generic thing to add (implementation being exactly what you have here.

drivers/virt/coco/arm-cca-host/arm-cca.c line 192

Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4 -0
+}
+
+static const struct platform_device_id arm_cca_host_id_table[] = {
+	{ RMI_DEV_NAME, 0},

Space before } and don't provide data until there is a use for it.

	{ RMI_DEV_NAME }

drivers/virt/coco/arm-cca-host/arm-cca.c line 196

Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4 -0
+	{ }
+};
+MODULE_DEVICE_TABLE(platform, arm_cca_host_id_table);
+

Consistency on spacing. I'd go for just 1 blank line for separation of things.

Quoted 13 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +13 -0
+
+static struct platform_driver cca_tsm_platform_driver = {
+	.probe = cca_tsm_probe,
+	.id_table = arm_cca_host_id_table,
+	.driver = {
+		.name = "cca_tsm",
+	},
+};
+
+MODULE_IMPORT_NS("PCI_IDE");
+module_platform_driver(cca_tsm_platform_driver);
+MODULE_DESCRIPTION("ARM CCA Host TSM driver");
+MODULE_LICENSE("GPL");
#42 lore raw

Re: [RFC PATCH v1 07/38] iommufd/viommu: Add support to associate viommu with kvm instance

In reply to Jonathan Cameron
Quoted 44 lines · Jonathan Cameron
Quoted 28 lines · On Mon, 28 Jul 2025 19:21:44 +0530 "Aneesh Kumar K.V (Arm)"…

The associated kvm instance will be used in later patch by iommufd to bind a tdi to kvm.

---
 drivers/iommu/iommufd/viommu.c | 45 +++++++++++++++++++++++++++++++++-
 include/linux/iommufd.h        |  3 +++
 include/uapi/linux/iommufd.h   | 12 +++++++++
 3 files changed, 59 insertions(+), 1 deletion(-)
drivers/iommu/iommufd/viommu.c +10 -0
index 2ca5809b238b..59f1e1176f7f 100644
@@ -2,6 +2,36 @@
 /* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES
  */
 #include "iommufd_private.h"
+#include "linux/tsm.h"
+
+#if IS_ENABLED(CONFIG_KVM)
+#include <linux/kvm_host.h>
+
+static int viommu_get_kvm(struct iommufd_viommu *viommu, int kvm_vm_fd)
+{
+	int rc = -EBADF;
+	struct file *filp;
+

drivers/iommu/iommufd/viommu.c line 24

Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10 -0
+	filp = fget(kvm_vm_fd);
+
+	if (!file_is_kvm(filp))
+		goto err_out;
+
+	/* hold the kvm reference via file descriptor */
+	viommu->kvm_filp = filp;
+	return 0;
+err_out:
+	viommu->kvm_filp = NULL;

Is this to undo side effects from this function on error?

Quoted 2 lines

kvm_filp is only set after all error paths so maybe this isn't needed?

Looks like you are right to me

Quoted 5 lines

If this isn't needed then use __free(fput) and no_free_ptr() to deal with filp more simply and in teh erorr path can just return -EBADF directly rather than the goto.

Or are we avoiding that stuff in iommufd?

Nope, gentle obvious use is fine :)

Jason

#59 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jonathan Cameron
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Is faux_device a better choice? I admit to not knowing entirely what it is for..

But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..

I would expect this to just call 'register tsm' ?

Jason

#92 lore raw

Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver

In reply to Jonathan Cameron
Quoted 12 lines · Jonathan Cameron
Quoted 12 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +11 -0
+}
 
+static void cca_tsm_remove(void *tsm_core)
+{
+	tsm_unregister(tsm_core);
+}
+
+static int cca_tsm_probe(struct platform_device *pdev)
+{
+	struct tsm_core_dev *tsm_core;
+
+	tsm_core = tsm_register(&pdev->dev, NULL, &cca_pci_ops);
Quoted 8 lines

drivers/virt/coco/arm-cca-host/arm-cca.c line 188

Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4 -0
+	if (IS_ERR(tsm_core))
+		return PTR_ERR(tsm_core);
+
+	return devm_add_action_or_reset(&pdev->dev, cca_tsm_remove, tsm_core);

So this makes two with the one in Dan's test code. devm_tsm_register() seems to be a useful generic thing to add (implementation being exactly what you have here.

Pelase no, this is insane, you have a probed driver with a probe/remove function pairing already. Why on earth would you use devm just to call a remove function :(

Just put tsm_unregister() in the normal driver remove like it is supposed to be done and use the drvdata to pass the tsm_core_dev pointer. It is easy and normal, look at fwctl for a very simple example.

devm is useful to solve complex things, these trivial things should be done normally..

Jason

#36 lore raw

Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE

In reply to Aneesh Kumar K.V (Arm)
Quoted 10 lines
drivers/vfio/pci/vfio_pci_core.c +1 -1
index 6328c3a05bcd..bee3cf3226e9 100644
@@ -1753,8 +1753,15 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
 	 * we need to request the region and the barmap tracks that.
 	 */
 	if (!vdev->barmap[index]) {
-		ret = pci_request_selected_regions(pdev,
+

drivers/vfio/pci/vfio_pci_core.c int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma · line 1766

Quoted 10 lines
drivers/vfio/pci/vfio_pci_core.c +7 -0
+		if (core_vdev->iommufd_device &&
+		    iommufd_device_need_exclusive_range(core_vdev->iommufd_device))
+			ret = pci_request_selected_regions_exclusive(pdev,
+							1 << index, "vfio-pci");
+		else
+			ret = pci_request_selected_regions(pdev,
 						   1 << index, "vfio-pci");
+
 		if (ret)
 			return ret;

I did't get the idea.

The purpose of my original patch [1] is not to make VFIO choose between pci_request_regions_exclusive() or pci_request_regions(). It is mainly to prevent userspace mmap/read/write against a vfio_cdev FD. For example:

If pci_request_selected_regions() is succesfully executed on mmap(), later TSM Bind would fail on its pci_request_regions_exclusive(). It means userspace should not mmap otherwise you can't do private assignment. Vice versa, if you've done TSM Bind, you cannot mmap anymore.

The _exclusive is just a bonus that further prevents "/dev/mem and the sysfs MMIO access"

[1]: https://lore.kernel.org/all/[email protected]/

Thanks,
Yilun

#32 lore raw

Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE

In reply to Jason Gunthorpe
Quoted 11 lines · Jason Gunthorpe
Quoted 10 lines · Aneesh Kumar K.V
Quoted 8 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)

Why would we need this?

I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

But also why is it an iommufd option? That doesn't seem right..

Jason

This is based on our previous discussion https://lore.kernel.org/all/[email protected]

I suggested a global option, this is a per-device option, and that especially seems wrong for iommufd. If it is per-device that is vfio,

I think this should be per-device. The original purpose of this pci_region_request_*() is to prevent further mmap/read/write against a vfio_cdev FD which would be used for private assignment. You shouldn't prevent all other devices from working with userspace APPs (e.g. DPDK) if there is one private assignment in system.

Quoted 9 lines

if it is global then vfio can pick it up during the early phases of opening the device.

Quoted 3 lines

IIUC, we intend to request the resource in exclusive mode for secure guests—regardless of whether the platform is Intel or ARM. Could you help clarify the MCE issue observed on Intel platforms in this context?

As I understand it Intel MCEs if the non-secure side ever reads from secure'd address space. So there is alot of emphasis there to ensure

Yeah, Intel TDX doesn't have a lower access control table for CC. So if host reads, the TLP sends and MCE happens.

Thanks,
Yilun

Quoted 3 lines

there are no CPU mappings.

Jason

author maintainer
#60 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jason Gunthorpe
Quoted 7 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Quoted 9 lines

Is faux_device a better choice? I admit to not knowing entirely what it is for..

But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..

I would expect this to just call 'register tsm' ?

The goal is to have tsm class device to be parented by the platform device.

# ls -al
total 0
drwxr-xr-x    2 root     root             0 Jan 13 06:07 .
drwxr-xr-x   23 root     root             0 Jan  1 00:00 ..
lrwxrwxrwx    1 root     root             0 Jan 13 06:07 tsm0 -> ../../devices/platform/arm-rmi-dev/tsm/tsm0
# pwd
/sys/class/tsm

-aneesh

author maintainer
#95 lore raw

Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver

In reply to Jonathan Cameron
Quoted 3 lines · Jonathan Cameron

On Mon, 28 Jul 2025 19:21:49 +0530
"Aneesh Kumar K.V (Arm)" <[email protected]> wrote:

...

Quoted 20 lines
Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5 -0
+
+#include "rmm-da.h"
+
+/* Number of streams that we can support at the hostbridge level */
+#define CCA_HB_PLATFORM_STREAMS 4

drivers/virt/coco/arm-cca-host/arm-cca.c line 27

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10 -0
+
+/* Total number of stream id supported at root port level */
+#define MAX_STREAM_ID	256
+
+DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T))
+static struct pci_tsm *cca_tsm_pci_probe(struct pci_dev *pdev)
+{
+	int rc;
+	struct pci_host_bridge *hb;
+	struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = NULL;

Read the stuff in cleanup.h and work out why this needs changing to be inline below and not use this NULL pattern here (unless you like grumpy Linus ;)

Quoted 5 lines

Note that with the err_out, even if you do that you'll still be breaking with the guidance doc (and actually causing undefined behavior :) Get rid of those gotos if you want to use __free()

I’ve already fixed up similar cases by removing the goto based on cleanup.h docs in other functions.I must have missed this one.

By the way, isn't using the NULL pattern acceptable when there are no additional lock variables involved (ie, unwind order doesn't matter)? Or should we always follow the pattern below regardless?

	struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) =
		vcalloc(sizeof(*dsc_pf0), GFP_KERNEL);

-aneesh

#10 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Jason Gunthorpe
Quoted 45 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe

arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56

Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5 -0
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
  	return crypt_ops->decrypt(addr, numpages);
  }
  EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;

Is this OK? I see code like this:

static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)

{
	if (force_dma_unencrypted(dev))
		return phys_to_dma_unencrypted(dev, phys);
	return phys_to_dma(dev, phys);

What are the ARM rules for generating dma addreses?

1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"

2) Device is T=1, memory is encrypted, use the phys_to_dma() normally

3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?

Quoted 5 lines
Quoted 2 lines

Can you document this in a comment?

If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.

No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.

Quoted 5 lines
Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

PCI SIG supports it, kernel should support it.

ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.

Rather than mapping the entire memory from the host, it would be ideal if the Coco vms have some sort of a callback to "make sure the DMA wouldn't fault for a device". e.g, it could be as simple as touching the page in Arm CCA (GFP_ZERO could do the trick, well one byte per Granule is good). or an ACCEPT a given page.

Is this a problem for AMDE SNP / Intel TDX ?

Suzuki

Quoted 2 lines

Jason

#93 lore raw

Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver

In reply to Jason Gunthorpe
Quoted 26 lines · On Tue, 29 Jul 2025 20:22:43 -0300 Jason Gunthorpe <[email protected]> wrote:
Quoted 12 lines · Jonathan Cameron
Quoted 12 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +11 -0
+}  
 
+static void cca_tsm_remove(void *tsm_core)
+{
+	tsm_unregister(tsm_core);
+}
+
+static int cca_tsm_probe(struct platform_device *pdev)
+{
+	struct tsm_core_dev *tsm_core;
+
+	tsm_core = tsm_register(&pdev->dev, NULL, &cca_pci_ops);
Quoted 8 lines

drivers/virt/coco/arm-cca-host/arm-cca.c line 188

Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +4 -0
+	if (IS_ERR(tsm_core))
+		return PTR_ERR(tsm_core);
+
+	return devm_add_action_or_reset(&pdev->dev, cca_tsm_remove, tsm_core);  

So this makes two with the one in Dan's test code. devm_tsm_register() seems to be a useful generic thing to add (implementation being exactly what you have here.

Pelase no, this is insane, you have a probed driver with a probe/remove function pairing already. Why on earth would you use devm just to call a remove function :(

Quoted 7 lines

Just put tsm_unregister() in the normal driver remove like it is supposed to be done and use the drvdata to pass the tsm_core_dev pointer. It is easy and normal, look at fwctl for a very simple example.

devm is useful to solve complex things, these trivial things should be done normally..

Sure, that would be fine for now. If we end up with a large complex flow that happens to have a tsm_register() in amongst various managed resources we can revisit. If they all end up looking like this then a manual call in remove is fine.

Quoted 2 lines

Jason

#61 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Aneesh Kumar K.V
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Is faux_device a better choice? I admit to not knowing entirely what it is for..

I'll go with a cautious yes to faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.

Not that many examples yet, but cpuidle-psci.c looks like a vaguely similar case to this one.

All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if probe() fails, which is cleaner in many ways than leaving a pointless stub behind.

Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.

+CC Greg on basis I may have wrong end of the stick ;)

Quoted 22 lines
Quoted 7 lines

But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..

I would expect this to just call 'register tsm' ?

The goal is to have tsm class device to be parented by the platform device.

# ls -al
total 0
drwxr-xr-x    2 root     root             0 Jan 13 06:07 .
drwxr-xr-x   23 root     root             0 Jan  1 00:00 ..
lrwxrwxrwx    1 root     root             0 Jan 13 06:07 tsm0 -> ../../devices/platform/arm-rmi-dev/tsm/tsm0
# pwd
/sys/class/tsm

-aneesh

#96 lore raw

Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver

In reply to Aneesh Kumar K.V
Quoted 37 lines · On Wed, 30 Jul 2025 14:28:55 +0530 Aneesh Kumar K.V <[email protected]>…
Quoted 3 lines · Jonathan Cameron

On Mon, 28 Jul 2025 19:21:49 +0530
"Aneesh Kumar K.V (Arm)" <[email protected]> wrote:

...

Quoted 20 lines
Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +5 -0
+
+#include "rmm-da.h"
+
+/* Number of streams that we can support at the hostbridge level */
+#define CCA_HB_PLATFORM_STREAMS 4

drivers/virt/coco/arm-cca-host/arm-cca.c line 27

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c new file +10 -0
+
+/* Total number of stream id supported at root port level */
+#define MAX_STREAM_ID	256
+
+DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T))
+static struct pci_tsm *cca_tsm_pci_probe(struct pci_dev *pdev)
+{
+	int rc;
+	struct pci_host_bridge *hb;
+	struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) = NULL;  

Read the stuff in cleanup.h and work out why this needs changing to be inline below and not use this NULL pattern here (unless you like grumpy Linus ;)

Quoted 5 lines

Note that with the err_out, even if you do that you'll still be breaking with the guidance doc (and actually causing undefined behavior :) Get rid of those gotos if you want to use __free()

I’ve already fixed up similar cases by removing the goto based on cleanup.h docs in other functions.I must have missed this one.

Quoted 6 lines

By the way, isn't using the NULL pattern acceptable when there are no additional lock variables involved (ie, unwind order doesn't matter)? Or should we always follow the pattern below regardless?

	struct cca_host_dsc_pf0 *dsc_pf0 __free(vfree) =
		vcalloc(sizeof(*dsc_pf0), GFP_KERNEL);

Always do this. It's not really about what happens today but more what we might break by failing to notice a future patch causes problems. Keeping the unwind ordering tightly couple with setup means we basically can't get it wrong (famous last words ;)

Jonathan

Quoted 2 lines

-aneesh

#98 lore raw

Re: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:50 +0530 "Aneesh Kumar K.V (Arm)"…

Create the realm physical device with RMM.

Hi Aneesh,

Various small things inline.

Jonathan

Quoted 30 lines
---
 arch/arm64/include/asm/rmi_cmds.h        |  31 +++++
 arch/arm64/include/asm/rmi_smc.h         |  72 ++++++++++-
 drivers/virt/coco/arm-cca-host/Makefile  |   2 +-
 drivers/virt/coco/arm-cca-host/arm-cca.c |  10 +-
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 150 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |   5 +
 6 files changed, 267 insertions(+), 3 deletions(-)
 create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.c
arch/arm64/include/asm/rmi_cmds.h +12 -0
index ef53147c1984..f0817bd3bab4 100644
@@ -505,4 +505,35 @@ static inline int rmi_rtt_unmap_unprotected(unsigned long rd,
 	return res.a0;
 }
 
+static inline unsigned long rmi_pdev_aux_count(unsigned long flags, u64 *aux_count)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_AUX_COUNT, flags, &res);
+
+	*aux_count = res.a1;
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_create(unsigned long pdev_phys,
+					    unsigned long pdev_params_phys)

arch/arm64/include/asm/rmi_cmds.h static inline int rmi_rtt_unmap_unprotected(unsigned long rd, · line 529

Quoted 10 lines
arch/arm64/include/asm/rmi_cmds.h +10 -0
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_CREATE,
+			     pdev_phys, pdev_params_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_get_state(unsigned long pdev_phys, unsigned long *state)

I'd use the enum for the state type and range check in here.

Quoted 15 lines
arch/arm64/include/asm/rmi_cmds.h +9 -0
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_GET_STATE, pdev_phys, &res);
+
+	*state = res.a1;
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +0 -0
index 42708d500048..a84ed61e5001 100644

arch/arm64/include/asm/rmi_smc.h line 29

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +0 -1
@@ -26,7 +26,7 @@
 #define SMC_RMI_DATA_CREATE		SMC_RMI_CALL(0x0153)
 #define SMC_RMI_DATA_CREATE_UNKNOWN	SMC_RMI_CALL(0x0154)
 #define SMC_RMI_DATA_DESTROY		SMC_RMI_CALL(0x0155)
-

Probably keep

arch/arm64/include/asm/rmi_smc.h line 29

Quoted 1 line
arch/arm64/include/asm/rmi_smc.h +1 -0
+#define SMC_RMI_PDEV_AUX_COUNT		SMC_RMI_CALL(0x0156)

and add one here as different groups.

Quoted 14 lines
arch/arm64/include/asm/rmi_smc.h +3 -0
 #define SMC_RMI_REALM_ACTIVATE		SMC_RMI_CALL(0x0157)
 #define SMC_RMI_REALM_CREATE		SMC_RMI_CALL(0x0158)
 #define SMC_RMI_REALM_DESTROY		SMC_RMI_CALL(0x0159)
@@ -47,6 +47,9 @@
 #define SMC_RMI_RTT_INIT_RIPAS		SMC_RMI_CALL(0x0168)
 #define SMC_RMI_RTT_SET_RIPAS		SMC_RMI_CALL(0x0169)
 
+#define SMC_RMI_PDEV_CREATE             SMC_RMI_CALL(0x0176)
+#define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
+
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0
 

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 274

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +1 -0
@@ -268,4 +271,71 @@ struct rec_run {
 	struct rec_exit exit;
 };
 
+enum rmi_pdev_state {

Type never used, but I think it should be. See above.

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 278

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4 -0
+	RMI_PDEV_NEW,
+	RMI_PDEV_NEEDS_KEY,
+	RMI_PDEV_HAS_KEY,
+	RMI_PDEV_READY,

Seems someone put another state in here, but I guess you'll update when the rest of the stack catches up.
RMI_PDEV_IDE_RESETTING,

Maybe throw a comment here for now.

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4 -0
+	RMI_PDEV_COMMUNICATING,
+	RMI_PDEV_STOPPING,
+	RMI_PDEV_STOPPED,
+	RMI_PDEV_ERROR,

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 292

Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10 -0
+};
+
+#define MAX_PDEV_AUX_GRANULES	32
+#define MAX_IOCOH_ADDR_RANGE	16
+#define MAX_FCOH_ADDR_RANGE	4
+
+#define RMI_PDEV_SPDM_TRUE	BIT(0)
+#define RMI_PDEV_IDE_TRUE	BIT(1)
+#define RMI_PDEV_FOCOH		BIT(2)
+#define RMI_PDEV_P2P_STREAM	BIT(3)

I'd stick flags in the name (assuming this is
RmiPDevFlags? I'm not sure as the spec I could
find has different usages other after BIT(!))

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 299

Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +7 -0
+
+#define RMI_HASH_SHA_256	0
+#define RMI_HASH_SHA_512	1
+
+struct rmi_pdev_addr_range {
+	unsigned long base;
+	unsigned long top;

Whilst we only care about platforms where this is u64, maybe just make that explicit so we can trivially see this matches the spec?

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 302

Quoted 3 lines
arch/arm64/include/asm/rmi_smc.h +3 -0
+};
+
+struct rmi_pdev_params {

Reference? Looks like B4.4.25 RmiPdevParams type in alp15. Though as there are some fields missing I guess this chagned.

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 315

Quoted 13 lines
arch/arm64/include/asm/rmi_smc.h +13 -0
+	union {
+		struct {
+			u64 flags;
+			u64 pdev_id;
+			u64 segment_id;
+			u64 ecam_addr;
+			u64 root_id;
+			u64 cert_id;
+			u64 rid_base;
+			u64 rid_top;
+			u64 hash_algo;
+			u64 num_aux;
+			u64 ide_sid;

Called ncoh_ide_side in the alpha 15. Maybe match that unless it is changing name in future version.

Quoted 13 lines
arch/arm64/include/asm/rmi_smc.h +13 -0
+			u64 ncoh_num_addr_range;
+			u64 coh_num_addr_range;
+		};
+		u8 padding1[0x100];
+	};
+
+	union { /* 0x100 */
+		u64 aux_granule[MAX_PDEV_AUX_GRANULES];
+		u8 padding2[0x100];
+	};
+
+	union { /* 0x200 */
+		struct {

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 338

Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10 -0
+			struct rmi_pdev_addr_range ncoh_addr_range[MAX_IOCOH_ADDR_RANGE];
+		};
+		u8 padding3[0x100];
+	};
+	union { /* 0x300 */
+		struct {
+			struct rmi_pdev_addr_range coh_addr_range[MAX_FCOH_ADDR_RANGE];
+		};
+		u8 padding4[0x100];
+	};

Maybe pad out the rest? Mostly so we can see here that it is 4k.

drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 127

Quoted 13 lines
arch/arm64/include/asm/rmi_smc.h +2 -0
+};
+
 #endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -1
index c8b0e6db1f47..84d97dd41191 100644
@@ -124,7 +124,15 @@ static int cca_tsm_connect(struct pci_dev *pdev)
 	rc = tsm_ide_stream_register(pdev, ide);
 	if (rc)
 		goto err_tsm;
-

Tidy up. I'd leave it.

Quoted 18 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +9 -0
+	/*
+	 * Take a module reference so that we won't call unregister
+	 * without rme_unasign_device
+	 */
+	if (!try_module_get(THIS_MODULE)) {
+		rc = -ENXIO;
+		goto err_tsm;
+	}
+	rme_asign_device(pdev);
 	/*
 	 * Once ide is setup enable the stream at endpoint
 	 * Root port will be done by RMM
drivers/virt/coco/arm-cca-host/rmm-da.c new file +0 -0
new file mode 100644
index 000000000000..426e530ac182

drivers/virt/coco/arm-cca-host/rmm-da.c line 41

Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4 -0
@@ -0,0 +1,150 @@
 
 
+
+static int init_pdev_params(struct pci_dev *pdev, struct rmi_pdev_params *params)
+{
+	void *aux;

Probably makes sense to reduce scope of aux to within the loop.

drivers/virt/coco/arm-cca-host/rmm-da.c line 45

Quoted 4 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4 -0
+	int rid, ret, i;
+	phys_addr_t aux_phys;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct pci_dev *rp = pcie_find_root_port(pdev);

Only used in one place for now anyway. Probably better to move it down there.

drivers/virt/coco/arm-cca-host/rmm-da.c line 48

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	struct pci_config_window *cfg = pdev->bus->sysdata;
+
+	dsc_pf0 = to_cca_dsc_pf0(pdev);

I'd do that at declaration. Seems little advantage in doing it down here.

drivers/virt/coco/arm-cca-host/rmm-da.c line 58

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+	/* assign the ep device with RMM */
+	rid = pci_dev_id(pdev);
+	params->pdev_id = rid;
+	/* slot number for certificate chain */
+	params->cert_id = 0;
+	/* io coherent spdm/ide and non p2p */
+	params->flags = RMI_PDEV_SPDM_TRUE | RMI_PDEV_IDE_TRUE;
+	params->ide_sid = dsc_pf0->sel_stream->stream_id;
+	params->hash_algo = RMI_HASH_SHA_256;
+	/* use the rid and MMIO resources of the epdev */

Spell out epdev or associate it with the pdev here more clearly.

drivers/virt/coco/arm-cca-host/rmm-da.c line 61

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	params->rid_top = params->rid_base = rid;
+	params->ecam_addr = cfg->res.start;
+	params->root_id = pci_dev_id(rp);
	params->root_id = pci_dev_id(pcie_find_root_port(pdev));

to me acts as better documentation fo what is going on than using the local variable rp.

drivers/virt/coco/arm-cca-host/rmm-da.c line 71

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+
+	params->ncoh_num_addr_range = pci_res_to_pdev_addr(params->ncoh_addr_range,
+							    ARRAY_SIZE(params->ncoh_addr_range),
+							    pdev->resource,
+							    DEVICE_COUNT_RESOURCE);
+
+	rmi_pdev_aux_count(params->flags, &params->num_aux);
+	pr_debug("%s using %ld pdev aux granules\n", __func__, (unsigned long)params->num_aux);
+	dsc_pf0->num_aux = params->num_aux;
+	for (i = 0; i < params->num_aux; i++) {
		void *aux = (void *)__get_free_page(GFP_KERNEL);
Quoted 9 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +9 -0
+		aux = (void *)__get_free_page(GFP_KERNEL);
+		if (!aux) {
+			ret = -ENOMEM;
+			goto err_free_aux;
+		}
+
+		aux_phys = virt_to_phys(aux);
+		if (rmi_granule_delegate(aux_phys)) {
+			ret = -ENXIO;

drivers/virt/coco/arm-cca-host/rmm-da.c line 90

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+			free_page((unsigned long)aux);
+			goto err_free_aux;
+		}
+		params->aux_granule[i] = aux_phys;
+		dsc_pf0->aux[i] = aux;
+	}
+	return 0;
+
+err_free_aux:
+	free_aux_pages(i, dsc_pf0->aux[i]);

I think you want

	free_aux_pages(i, dsc_pf0->aux);

Assuming this is supposed to unwind the loop above.

drivers/virt/coco/arm-cca-host/rmm-da.c line 93

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	return -ENOMEM;
+}
+

Trivial: Single line probably fine here.

drivers/virt/coco/arm-cca-host/rmm-da.c line 104

Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11 -0
+
+int rme_asign_device(struct pci_dev *pci_dev)
+{
+	int ret;
+	void *rmm_pdev;
+	unsigned long state;
+	phys_addr_t rmm_pdev_phys;
+	struct rmi_pdev_params *params;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+
+	dsc_pf0 = to_cca_dsc_pf0(pci_dev);

Might as well save a line with

	struct cca_host_dsc_pf0 *dsc_pf0 = to_cca_dsc_pf0(pci_dev);

drivers/virt/coco/arm-cca-host/rmm-da.c line 107

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	rmm_pdev = (void *)get_zeroed_page(GFP_KERNEL);
+	if (!rmm_pdev) {
+		ret = -ENOMEM;

return -ENOMEM;

Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11 -0
+		goto err_out;
+	}
+
+	rmm_pdev_phys = virt_to_phys(rmm_pdev);
+	if (rmi_granule_delegate(rmm_pdev_phys)) {
+		ret = -ENXIO;
+		goto err_free_pdev;
+	}
+
+	params = (struct rmi_pdev_params *)get_zeroed_page(GFP_KERNEL);
+	if (!params) {

drivers/virt/coco/arm-cca-host/rmm-da.c line 128

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+		ret = -ENOMEM;
+		goto err_granule_undelegate;
+	}
+
+	ret = init_pdev_params(pci_dev, params);
+	if (ret)
+		goto err_free_params;
+
+	ret = rmi_pdev_create(rmm_pdev_phys, virt_to_phys(params));
+	pr_info("rmi_pdev_create(0x%llx, 0x%llx): %d\n", rmm_pdev_phys, virt_to_phys(params), ret);

RFC, but even so I'd demote to debug and use dynamic debug stuff to enable them for your testing.

drivers/virt/coco/arm-cca-host/rmm-da.c line 134

Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +6 -0
+	if (ret)
+		goto err_free_aux;
+
+	rmi_pdev_get_state(rmm_pdev_phys, &state);
+	if (state != RMI_PDEV_NEW)
+		goto err_free_aux;

Nothing to unwind in rmi_pdev_create()? We've told the RMM about it then we blow away it's resources. Seems unwise! Maybe this is cleaned up elsewhere but if so a comment is probably good.

Quoted 4 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4 -0
+
+	dsc_pf0->rmm_pdev = rmm_pdev;
+	free_page((unsigned long)params);
+	return 0;

drivers/virt/coco/arm-cca-host/rmm-da.c line 148

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+
+err_free_aux:
+	free_aux_pages(dsc_pf0->num_aux, dsc_pf0->aux);
+err_free_params:
+	free_page((unsigned long)params);
+err_granule_undelegate:
+	rmi_granule_undelegate(rmm_pdev_phys);
+err_free_pdev:
+	free_page((unsigned long)rmm_pdev);
+err_out:

One of my favourite nitpicks. Why not just return if nothing to do? That tends to save a reviewer a bit of scrolling when checking error paths do correct unwinding.

Quoted 23 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2 -0
+	return ret;
+}
drivers/virt/coco/arm-cca-host/rmm-da.h +5 -0
index 840cb584acdd..179ba68f2430 100644
@@ -14,6 +14,10 @@
 struct cca_host_dsc_pf0 {
 	struct pci_tsm_pf0 pci;
 	struct pci_ide *sel_stream;
+
+	void *rmm_pdev;
+	int num_aux;
+	void *aux[MAX_PDEV_AUX_GRANULES];
 };
 
 static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
@@ -26,4 +30,5 @@ static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
 	return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm);
 }
 
+int rme_asign_device(struct pci_dev *pdev);
 #endif
#62 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jonathan Cameron
Quoted 28 lines · On Wed, 30 Jul 2025 11:38:27 +0100 Jonathan Cameron…
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Is faux_device a better choice? I admit to not knowing entirely what it is for..

I'll go with a cautious yes to faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.

Not that many examples yet, but cpuidle-psci.c looks like a vaguely similar case to this one.

All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if probe() fails, which is cleaner in many ways than leaving a pointless stub behind.

Quoted 4 lines

Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.

+CC Greg on basis I may have wrong end of the stick ;)

This time with at least one less typo in Greg's email address.

Quoted 25 lines
Quoted 23 lines
Quoted 8 lines

But alternatively, why do we need a dummy "hw" struct device at all? Typically a subsystem like TSM should be structured to create its own struct devices..

I would expect this to just call 'register tsm' ?

The goal is to have tsm class device to be parented by the platform device.

# ls -al
total 0
drwxr-xr-x    2 root     root             0 Jan 13 06:07 .
drwxr-xr-x   23 root     root             0 Jan  1 00:00 ..
lrwxrwxrwx    1 root     root             0 Jan 13 06:07 tsm0 -> ../../devices/platform/arm-rmi-dev/tsm/tsm0
# pwd
/sys/class/tsm

-aneesh

maintainer
#63 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

Quoted 31 lines · Jonathan Cameron
Quoted 28 lines · On Wed, 30 Jul 2025 11:38:27 +0100 Jonathan Cameron…
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Is faux_device a better choice? I admit to not knowing entirely what it is for..

I'll go with a cautious yes to faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.

Not that many examples yet, but cpuidle-psci.c looks like a vaguely similar case to this one.

All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if probe() fails, which is cleaner in many ways than leaving a pointless stub behind.

Quoted 5 lines
Quoted 4 lines

Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.

+CC Greg on basis I may have wrong end of the stick ;)

This time with at least one less typo in Greg's email address.

Yes, use faux_device if you need/want a struct device to represent something in the tree and it does NOT have any real platform resources behind it. That's explicitly what it was designed for.

thanks,

greg k-h

#104 lore raw

Re: [RFC PATCH v1 14/38] coco: host: arm64: Device communication support

In reply to Aneesh Kumar K.V (Arm)
Quoted 45 lines · On Mon, 28 Jul 2025 19:21:51 +0530 "Aneesh Kumar K.V (Arm)"…

Add helpers for device communication from RMM

---
 arch/arm64/include/asm/rmi_cmds.h        |  11 ++
 arch/arm64/include/asm/rmi_smc.h         |  49 ++++++
 drivers/virt/coco/arm-cca-host/arm-cca.c |  45 ++++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 198 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  41 +++++
 5 files changed, 344 insertions(+)

#endif /* __ASM_RMI_CMDS_H */

arch/arm64/include/asm/rmi_smc.h +25 -0
index a84ed61e5001..8bece465b670 100644
@@ -47,6 +47,7 @@
 
 
+#define RMI_DEV_COMM_EXIT_CACHE_REQ	BIT(0)
+#define RMI_DEV_COMM_EXIT_CACHE_RSP	BIT(1)
+#define RMI_DEV_COMM_EXIT_SEND		BIT(2)
+#define RMI_DEV_COMM_EXIT_WAIT		BIT(3)
+#define RMI_DEV_COMM_EXIT_MULTI		BIT(4)
+
+#define RMI_DEV_COMM_NONE	0
+#define RMI_DEV_COMM_RESPONSE	1
+#define RMI_DEV_COMM_ERROR	2
+
+#define RMI_PROTOCOL_SPDM		0
+#define RMI_PROTOCOL_SECURE_SPDM	1
+
+#define RMI_DEV_VCA			0
+#define RMI_DEV_CERTIFICATE		1
+#define RMI_DEV_MEASUREMENTS		2
+#define RMI_DEV_INTERFACE_REPORT	3
+
+struct rmi_dev_comm_enter {
+	u64 status;
+	u64 req_addr;
+	u64 resp_addr;
+	u64 resp_len;
+};
+

arch/arm64/include/asm/rmi_smc.h struct rmi_pdev_params { · line 376

Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10 -0
+struct rmi_dev_comm_exit {
+	u64 flags;
+	u64 cache_req_offset;
+	u64 cache_req_len;
+	u64 cache_rsp_offset;
+	u64 cache_rsp_len;
+	u64 cache_obj_id;
+	u64 protocol;
+	u64 req_len;
+	u64 timeout;

In latest spec called rsp_timeout.
Not sure we care that much but if no strong reason otherwise, should aim to match the spec text. (Maybe this got renamed?)

Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +1 -0
+};
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
index 84d97dd41191..294a6ef60d5f 100644

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 92

Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5 -0
@@ -85,6 +85,45 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)
 	vfree(dsc_pf0);
 }
 
+static int init_dev_communication_buffers(struct cca_host_comm_data *comm_data)
+{
+	int ret = -ENOMEM;
+
+	comm_data->io_params = (struct rmi_dev_comm_data *)get_zeroed_page(GFP_KERNEL);

Hmm. There isn't a DEFINE_FREE() yet for free_page(). Maybe time to add one. If we did then we'd use local variables until all allocations succeed then assign with no_free_ptr()

Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5 -0
+	if (!comm_data->io_params)
+		goto err_out;
+
+	comm_data->resp_buff = (void *)__get_free_page(GFP_KERNEL);
+	if (!comm_data->resp_buff)

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 107

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10 -0
+		goto err_res_buff;
+
+	comm_data->req_buff = (void *)__get_free_page(GFP_KERNEL);
+	if (!comm_data->req_buff)
+		goto err_req_buff;
+
+
+	comm_data->io_params->enter.status = RMI_DEV_COMM_NONE;
+	comm_data->io_params->enter.resp_addr = virt_to_phys(comm_data->resp_buff);
+	comm_data->io_params->enter.req_addr  = virt_to_phys((void *)comm_data->req_buff);

I think it's already a a void * and even if it were some other pointer type no cast would be necessary.

Quoted 26 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +14 -0
+	comm_data->io_params->enter.resp_len = 0;
+
+	return 0;
+
+err_req_buff:
+	free_page((unsigned long)comm_data->resp_buff);
+err_res_buff:
+	free_page((unsigned long)comm_data->io_params);
+err_out:
+	return ret;
+}
+
 
+
 /* per root port unique with multiple restrictions. For now global */
 static DECLARE_BITMAP(cca_stream_ids, MAX_STREAM_ID);
@@ -124,6 +163,7 @@ static int cca_tsm_connect(struct pci_dev *pdev)
 	rc = tsm_ide_stream_register(pdev, ide);
 	if (rc)
 		goto err_tsm;
+	init_dev_communication_buffers(&dsc_pf0->comm_data);
 	/*
 	 * Take a module reference so that we won't call unregister
 	 * without rme_unasign_device

drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 175

Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
@@ -133,6 +173,11 @@ static int cca_tsm_connect(struct pci_dev *pdev)
 		goto err_tsm;
 	}
 	rme_asign_device(pdev);

rme_assign_device() - I obviously missed this earlier!

drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 177

Quoted 2 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +2 -0
+	/*
+	 * Schedule a work to fetch device certificate and setup IDE

Single line comment probably fine here. Though it perhaps doesn't add much over the function name.

Quoted 16 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +3 -0
+	 */
+	schedule_rme_ide_setup(pdev);
+
 	/*
 	 * Once ide is setup enable the stream at endpoint
 	 * Root port will be done by RMM
drivers/virt/coco/arm-cca-host/rmm-da.c +2 -0
index 426e530ac182..d123940ce82e 100644
@@ -148,3 +148,201 @@ int rme_asign_device(struct pci_dev *pci_dev)
 err_out:
 	return ret;
 }
+
+static int doe_send_req_resp(struct pci_tsm *tsm)

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 162

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+{
+	u8 protocol;
+	int ret, data_obj_type;
+	struct cca_host_comm_data *comm_data;
+	struct rmi_dev_comm_exit *io_exit;
+
+	comm_data = to_cca_comm_data(tsm->pdev);
+
+	io_exit = &comm_data->io_params->exit;
+	protocol = io_exit->protocol;

For all these I'd combine with the declarations.

Quoted 24 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +24 -0
+
+	pr_debug("doe_req size:%lld doe_io_type=%d\n", io_exit->req_len, (int)protocol);
+
+	if (protocol == RMI_PROTOCOL_SPDM)
+		data_obj_type = PCI_DOE_PROTO_CMA;
+	else if (protocol == RMI_PROTOCOL_SECURE_SPDM)
+		data_obj_type = PCI_DOE_PROTO_SSESSION;
+	else
+		return -EINVAL;
+
+	ret = pci_tsm_doe_transfer(tsm->dsm_dev, data_obj_type,
+				   comm_data->req_buff, io_exit->req_len,
+				   comm_data->resp_buff, PAGE_SIZE);
+	pr_debug("doe returned:%d\n", ret);
+	return ret;
+}
+
+/* Parallel update for cca_dsc contents FIXME!! */
+static int __do_dev_communicate(int type, struct pci_tsm *tsm)
+{
+	int ret;
+	bool is_multi;
+	u8 *cache_buf;
+	int *cache_offset;

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 196

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+	int nbytes, cache_remaining;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct rmi_dev_comm_exit *io_exit;
+	struct rmi_dev_comm_enter *io_enter;
+	struct cca_host_comm_data *comm_data;
+
+
+	comm_data = to_cca_comm_data(tsm->pdev);
+	io_enter = &comm_data->io_params->enter;
+	io_exit = &comm_data->io_params->exit;

Might as well set these local variables as the declaration point above. None of them will be very long lines.

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 206

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->dsm_dev);
+redo_communicate:
+	is_multi = false;
+
+	if (type == PDEV_COMMUNICATE)
+		ret = rmi_pdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
+					   virt_to_phys(comm_data->io_params));
+	else
+		ret = RMI_ERROR_INPUT;

I'd split this case out and return here farther than using the match below as it feels like the error message auto to be more specific. Something about type not matching.

Quoted 31 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31 -0
+	if (ret != RMI_SUCCESS) {
+		pr_err("pdev communicate error\n");
+		return ret;
+	}
+
+	/* caching request from RMM */
+	if (io_exit->flags & RMI_DEV_COMM_EXIT_CACHE_RSP) {
+		switch (io_exit->cache_obj_id) {
+		case RMI_DEV_VCA:
+			cache_buf = dsc_pf0->vca.buf;
+			cache_offset = &dsc_pf0->vca.size;
+			cache_remaining = sizeof(dsc_pf0->vca.buf) - *cache_offset;
+			break;
+		case RMI_DEV_CERTIFICATE:
+			cache_buf = dsc_pf0->cert_chain.cache.buf;
+			cache_offset = &dsc_pf0->cert_chain.cache.size;
+			cache_remaining = sizeof(dsc_pf0->cert_chain.cache.buf) - *cache_offset;
+			break;
+		default:
+			/* FIXME!! depending on the DevComms status,
+			 * it might require to ABORT the communcation.
+			 */
+			return -EINVAL;
+		}
+
+		if (io_exit->cache_rsp_len > cache_remaining)
+			return -EINVAL;
+
+		memcpy(cache_buf + *cache_offset,
+		       (comm_data->resp_buff + io_exit->cache_rsp_offset), io_exit->cache_rsp_len);
+		*cache_offset += io_exit->cache_rsp_len;

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 247

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+	}
+
+	/*
+	 * wait for last packet request from RMM.
+	 * We should not find this because our device communication in synchronous
+	 */
+	if (io_exit->flags & RMI_DEV_COMM_EXIT_WAIT)
+		return -ENXIO;
+
+	is_multi = !!(io_exit->flags & RMI_DEV_COMM_EXIT_MULTI);

!! doesn't add anything here that I can see over

	is_multi = io_exit->flags & RMI_DEV_COMM_EXIT_MULTI;
Quoted 31 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31 -0
+
+	/* next packet to send */
+	if (io_exit->flags & RMI_DEV_COMM_EXIT_SEND) {
+		nbytes = doe_send_req_resp(tsm);
+		if (nbytes < 0) {
+			/* report error back to RMM */
+			io_enter->status = RMI_DEV_COMM_ERROR;
+		} else {
+			/* send response back to RMM */
+			io_enter->resp_len = nbytes;
+			io_enter->status = RMI_DEV_COMM_RESPONSE;
+		}
+	} else {
+		/* no data transmitted => no data received */
+		io_enter->resp_len = 0;
+	}
+
+	/* The call need to do multiple request/respnse */
+	if (is_multi)
+		goto redo_communicate;
+
+	return 0;
+}
+
+static int do_dev_communicate(int type, struct pci_tsm *tsm, int target_state)
+{
+	int ret;
+	unsigned long state;
+	unsigned long error_state;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct rmi_dev_comm_enter *io_enter;

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 288

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->dsm_dev);
+	io_enter = &dsc_pf0->comm_data.io_params->enter;
+	io_enter->resp_len = 0;
+	io_enter->status = RMI_DEV_COMM_NONE;
+
+	state = -1;
+	do {
+		ret = __do_dev_communicate(type, tsm);
+		if (ret != 0) {

if (ret)

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 290

Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +2 -0
+			pr_err("dev communication error\n");
+			break;

I'd just return in error cases.

Quoted 17 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +17 -0
+		}
+
+		if (type == PDEV_COMMUNICATE) {
+			ret = rmi_pdev_get_state(virt_to_phys(dsc_pf0->rmm_pdev),
+						 &state);
+			error_state = RMI_PDEV_ERROR;
+		}
+		if (ret != 0) {
+			pr_err("Get dev state error\n");
+			break;
+		}
+	} while (state != target_state && state != error_state);
+
+	pr_info("dev_io_complete: status: %d state:%ld\n", ret, state);
+
+	return state;
+}
#109 lore raw

Re: [RFC PATCH v1 15/38] coco: host: arm64: Stop and destroy the physical device

In reply to Aneesh Kumar K.V (Arm)
Quoted 1 line · On Mon, 28 Jul 2025 19:21:52 +0530 "Aneesh Kumar K.V (Arm)"…

Add support for stopping and destroying physical devices.

I think it's an odd mix to not do create and destroy in a single patch. Same with start and stop.
Leaves reviewers thinking perhaps you weren't cleaning up properly an any error paths are much less obvious.

Quoted 79 lines
---
 arch/arm64/include/asm/rmi_cmds.h        | 18 ++++++++++++++++++
 arch/arm64/include/asm/rmi_smc.h         |  2 ++
 drivers/virt/coco/arm-cca-host/arm-cca.c |  3 +++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 21 +++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  1 +
 5 files changed, 45 insertions(+)
arch/arm64/include/asm/rmi_cmds.h +18 -0
index eb0034a675bb..d4ea9f8363f5 100644
@@ -547,4 +547,22 @@ static inline unsigned long rmi_pdev_communicate(unsigned long pdev_phys,
 	return res.a0;
 }
 
+static inline unsigned long rmi_pdev_stop(unsigned long pdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_STOP, pdev_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_destroy(unsigned long pdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_DESTROY, pdev_phys, &res);
+
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +2 -0
index 8bece465b670..9f25a876238e 100644
@@ -49,7 +49,9 @@
 
 #define SMC_RMI_PDEV_COMMUNICATE        SMC_RMI_CALL(0x0175)
 #define SMC_RMI_PDEV_CREATE             SMC_RMI_CALL(0x0176)
+#define SMC_RMI_PDEV_DESTROY		SMC_RMI_CALL(0x0177)
 #define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
+#define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
 
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0
drivers/virt/coco/arm-cca-host/arm-cca.c +3 -0
index 294a6ef60d5f..c65b81f0706f 100644
@@ -210,12 +210,15 @@ static void cca_tsm_disconnect(struct pci_dev *pdev)
 	ide = dsc_pf0->sel_stream;
 	dsc_pf0->sel_stream = NULL;
 	pci_ide_stream_disable(pdev, ide);
+	rme_unassign_device(pdev);
+	module_put(THIS_MODULE);
 	tsm_ide_stream_unregister(ide);
 	pci_ide_stream_teardown(rp, ide);
 	pci_ide_stream_teardown(pdev, ide);
 	pci_ide_stream_unregister(ide);
 	clear_bit(ide->stream_id, cca_stream_ids);
 	pci_ide_stream_free(ide);
+	free_dev_communication_buffers(&dsc_pf0->comm_data);
 }
 
 static const struct pci_tsm_ops cca_pci_ops = {
drivers/virt/coco/arm-cca-host/rmm-da.c +0 -0
index d123940ce82e..ec8c5bfcee35 100644
@@ -346,3 +346,24 @@ int schedule_rme_ide_setup(struct pci_dev *pdev)
 
 	return 0;
 }

drivers/virt/coco/arm-cca-host/rmm-da.c int schedule_rme_ide_setup(struct pci_dev *pdev) · line 358

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+
+void rme_unassign_device(struct pci_dev *pdev)
+{
+	unsigned long ret;
+	unsigned long state;
+	phys_addr_t rmm_pdev_phys;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+
+	dsc_pf0 = to_cca_dsc_pf0(pdev);
+	rmm_pdev_phys = virt_to_phys(dsc_pf0->rmm_pdev);

As with previous patches, I'd set as many of these are seems reasonable at the variable declations.

drivers/virt/coco/arm-cca-host/rmm-da.c int schedule_rme_ide_setup(struct pci_dev *pdev) · line 364

Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +6 -0
+	ret = rmi_pdev_stop(rmm_pdev_phys);
+	if (WARN_ON(ret != RMI_SUCCESS))
+		return;
+
+	state = do_pdev_communicate(pdev->tsm, RMI_PDEV_STOPPED);
+	/* ignore the error state and destroy the device */

WARN_ON is rather heavy if you want to ignore it.

Quoted 16 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +5 -0
+	WARN_ON(state != RMI_PDEV_STOPPED);
+	ret = rmi_pdev_destroy(rmm_pdev_phys);
+	if (WARN_ON(ret != RMI_SUCCESS))
+		return;
+}
drivers/virt/coco/arm-cca-host/rmm-da.h +1 -0
index b9ddc4d9112b..c401be55d770 100644
@@ -71,5 +71,6 @@ static inline struct cca_host_comm_data *to_cca_comm_data(struct pci_dev *pdev)
 }
 
 int rme_asign_device(struct pci_dev *pdev);
+void rme_unassign_device(struct pci_dev *pdev);
 int schedule_rme_ide_setup(struct pci_dev *pdev);
 #endif
#116 lore raw

Re: [RFC PATCH v1 19/38] coco: host: arm64: set_pubkey support

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:56 +0530 "Aneesh Kumar K.V (Arm)"…

Add changes to share the device's public key with the RMM.

A few minor comments inline.

Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +15 -0
index ec8c5bfcee35..3715e6d58c83 100644
@@ -6,6 +6,9 @@
 #include <linux/pci.h>
 #include <linux/pci-ecam.h>
 #include <asm/rmi_cmds.h>
+#include <crypto/internal/rsa.h>
+#include <keys/asymmetric-type.h>
+#include <keys/x509-parser.h>
 
 #include "rmm-da.h"
 
@@ -311,6 +314,136 @@ static int do_pdev_communicate(struct pci_tsm *tsm, int target_state)
 	return do_dev_communicate(PDEV_COMMUNICATE, tsm, target_state);
 }
 
+static int parse_certificate_chain(struct pci_tsm *tsm)
+{
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	unsigned int chain_size;
+	unsigned int offset = 0;
+	u8 *chain_data;
+	int ret = 0;
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->pdev);
+	chain_size = dsc_pf0->cert_chain.cache.size;
+	chain_data = dsc_pf0->cert_chain.cache.buf;
+

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 338

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+	while (offset < chain_size) {
+		unsigned int cert_len =
+			x509_get_certificate_length(chain_data + offset,
+						    chain_size - offset);
+		struct x509_certificate *cert =
+			x509_cert_parse(chain_data + offset, cert_len);
+
+		if (IS_ERR(cert)) {
+			pr_warn("%s(): parsing of certificate chain not successful\n", __func__);
+			ret = PTR_ERR(cert);

Direct return looks fine here. Maybe add a DEFINE_FREE(x509_cert,...) as then can use direct returns throughout.

Quoted 23 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +23 -0
+			break;
+		}
+
+		if (offset + cert_len == chain_size) {
+			dsc_pf0->cert_chain.public_key = kzalloc(cert->pub->keylen, GFP_KERNEL);
+			if (!dsc_pf0->cert_chain.public_key) {
+				ret = -ENOMEM;
+				x509_free_certificate(cert);
+				break;
+			}
+
+			if (!strcmp("ecdsa-nist-p256", cert->pub->pkey_algo)) {
+				dsc_pf0->rmi_signature_algorithm = RMI_SIG_ECDSA_P256;
+			} else if (!strcmp("ecdsa-nist-p384", cert->pub->pkey_algo)) {
+				dsc_pf0->rmi_signature_algorithm = RMI_SIG_ECDSA_P384;
+			} else if (!strcmp("rsa", cert->pub->pkey_algo)) {
+				dsc_pf0->rmi_signature_algorithm = RMI_SIG_RSASSA_3072;
+			} else {
+				ret = -ENXIO;
+				x509_free_certificate(cert);
+				break;
+			}
+			memcpy(dsc_pf0->cert_chain.public_key, cert->pub->key, cert->pub->keylen);

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 371

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+			dsc_pf0->cert_chain.public_key_size = cert->pub->keylen;
+		}
+
+		x509_free_certificate(cert);
+
+		offset += cert_len;
+	}
+
+	if (ret == 0)
+		dsc_pf0->cert_chain.valid = true;
	if (ret)
		return ret;

	dsc_pf0->cert_chain.valid = true;

	return 0;

would be my preference for style here but others may disagree.

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 379

Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +8 -0
+
+	return ret;
+}
+
+static int pdev_set_public_key(struct pci_tsm *tsm)
+{
+	struct rmi_public_key_params *key_shared;
+	unsigned long expected_key_len = 0;

Don't set this. It's only used in places where it is explicitly set and if it is used anywhere else we want the compiler to tell us.

Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +30 -0
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	int ret;
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->pdev);
+	/* Check that all the necessary information was captured from communication */
+	if (!dsc_pf0->cert_chain.valid)
+		return -EINVAL;
+
+	key_shared = (struct rmi_public_key_params *)get_zeroed_page(GFP_KERNEL);
+	if (!key_shared)
+		return -ENOMEM;
+
+	key_shared->rmi_signature_algorithm = dsc_pf0->rmi_signature_algorithm;
+
+	switch (key_shared->rmi_signature_algorithm) {
+	case RMI_SIG_ECDSA_P384:
+		expected_key_len = 97;
+
+		if (dsc_pf0->cert_chain.public_key_size != expected_key_len)
+			return -EINVAL;
+		key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size;
+		memcpy(key_shared->public_key,
+		       dsc_pf0->cert_chain.public_key,
+		       dsc_pf0->cert_chain.public_key_size);
+		key_shared->metadata_len = 0;
+		break;
+	case RMI_SIG_ECDSA_P256:
+		expected_key_len = 65;
+
+		if (dsc_pf0->cert_chain.public_key_size != expected_key_len)

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 419

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+			return -EINVAL;
+		key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size;
+		memcpy(key_shared->public_key,
+		       dsc_pf0->cert_chain.public_key,
+		       dsc_pf0->cert_chain.public_key_size);
+		key_shared->metadata_len = 0;
+		break;
+	case RMI_SIG_RSASSA_3072:
+		expected_key_len = 385;
+		struct rsa_key rsa_key = {0};

Shouldn't define this inline. Maybe move up a line and add some {} to set the scope to this case statement.

Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +5 -0
+		int ret_rsa_parse = rsa_parse_pub_key(&rsa_key,
+						      dsc_pf0->cert_chain.public_key,
+						      dsc_pf0->cert_chain.public_key_size);
+		/* This also checks the key_len */
+		if (ret_rsa_parse)

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 434

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+			return ret_rsa_parse;
+		/*
+		 * exponent is usally 65537 (size = 24bits) but in rare cases
+		 * it size can be as large as the modulus
+		 */
+		if (rsa_key.e_sz > expected_key_len)
+			return -EINVAL;
+		key_shared->public_key_len = rsa_key.n_sz;
+		key_shared->metadata_len = rsa_key.e_sz;
+		memcpy(key_shared->public_key, (unsigned char *)rsa_key.n, rsa_key.n_sz);

Why is the cast needed?

Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +11 -0
+		memcpy(key_shared->metadata, (unsigned char *)rsa_key.e, rsa_key.e_sz);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = rmi_pdev_set_pubkey(virt_to_phys(dsc_pf0->rmm_pdev),
+				  virt_to_phys(key_shared));
+	free_page((unsigned long)key_shared);
+	return ret;
+}
#120 lore raw

Re: [RFC PATCH v1 20/38] coco: host: arm64: Add support for creating a virtual device

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:57 +0530 "Aneesh Kumar K.V (Arm)"…

Changes to support the creation of virtual device objects with RMM.

Hi Aneesh,

Really trivial stuff in this one.

arch/arm64/include/asm/rmi_smc.h line 57

Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +2 -0
index 4a5ba98c1c0d..e5238b271493 100644
@@ -53,6 +53,8 @@
 #define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
 #define SMC_RMI_PDEV_SET_PUBKEY		SMC_RMI_CALL(0x017b)
 #define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
+#define SMC_RMI_VDEV_CREATE		SMC_RMI_CALL(0x0187)
+

One blankl line probably enough.

arch/arm64/include/asm/rmi_smc.h line 59

Quoted 2 lines
arch/arm64/include/asm/rmi_smc.h +0 -0
 #define RMI_ABI_MAJOR_VERSION	1

t/coco/arm-cca-host/arm-cca.c b/drivers/virt/coco/arm-cca-host/arm-cca.c

Quoted 13 lines

index c65b81f0706f..2da513f45974 100644

drivers/virt/coco/arm-cca-host/arm-cca.c +2 -0
@@ -10,6 +10,8 @@
 #include <linux/pci.h>
 #include <linux/tsm.h>
 #include <linux/vmalloc.h>
+#include <linux/kvm_host.h>
+#include <linux/pci.h>
 
 #include "rmm-da.h"
 

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_disconnect(struct pci_dev *pdev) · line 230

Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5 -0
@@ -221,11 +223,39 @@ static void cca_tsm_disconnect(struct pci_dev *pdev)
 	free_dev_communication_buffers(&dsc_pf0->comm_data);
 }
 
+static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_dev,
+				    struct kvm *kvm, u64 tdi_id)
+{
+	void *rmm_vdev;
+	struct cca_host_tdi *host_tdi __free(kfree) = NULL;

Move declaration and registration of destructor down to where it's constructed. (Follow guidance in cleanup.h.

Quoted 18 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +18 -0
+	struct realm *realm = &kvm->arch.realm;
+
+	if (pdev->is_virtfn)
+		return ERR_PTR(-ENXIO);
+
+	if (!try_module_get(THIS_MODULE))
+		return ERR_PTR(-ENXIO);
+
+	host_tdi = kmalloc(sizeof(struct cca_host_tdi), GFP_KERNEL);
+	if (!host_tdi)
+		return ERR_PTR(-ENOMEM);
+
+	rmm_vdev = rme_create_vdev(realm, pdev, pf0_dev, tdi_id);
+	if (!IS_ERR_OR_NULL(rmm_vdev)) {
+		host_tdi->rmm_vdev = rmm_vdev;
+		return &no_free_ptr(host_tdi)->tdi;
+	}
+

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_disconnect(struct pci_dev *pdev) · line 258

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5 -0
+	module_put(THIS_MODULE);
+	return rmm_vdev;
+}
+
 static const struct pci_tsm_ops cca_pci_ops = {
 	.probe = cca_tsm_pci_probe,
 	.remove = cca_tsm_pci_remove,
 	.connect = cca_tsm_connect,
 	.disconnect = cca_tsm_disconnect,
+	.bind	= cca_tsm_bind,

Odd spacing. Seems just bind is using a tab.

Quoted 1 line
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
 };
#122 lore raw

Re: [RFC PATCH v1 21/38] coco: host: arm64: Add support for virtual device communication

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:58 +0530 "Aneesh Kumar K.V (Arm)"…

Add support for vdev_communicate with RMM.

One minor comment.

drivers/virt/coco/arm-cca-host/rmm-da.c static int __do_dev_communicate(int type, struct pci_tsm *tsm)

Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +0 -0
index 41314db1d568..8635f361bbe8 100644
@@ -207,8 +207,14 @@ static int __do_dev_communicate(int type, struct pci_tsm *tsm)

Can type parameter be an enum so we can see all cases are covered?

Quoted 12 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +7 -2
 	if (type == PDEV_COMMUNICATE)
 		ret = rmi_pdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
 					   virt_to_phys(comm_data->io_params));
-	else
-		ret = RMI_ERROR_INPUT;
+	else {
+		struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi);
+
+		ret = rmi_vdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
+					   virt_to_phys(host_tdi->rmm_vdev),
+					   virt_to_phys(comm_data->io_params));
+	}
#125 lore raw

Re: [RFC PATCH v1 22/38] coco: host: arm64: Stop and destroy virtual device

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:59 +0530 "Aneesh Kumar K.V (Arm)"…

Add support for vdev_stop and vdev_destroy.

Really trivial comments.

Quoted 26 lines
---
 arch/arm64/include/asm/rmi_cmds.h        | 21 ++++++++
 arch/arm64/include/asm/rmi_smc.h         |  3 +-
 drivers/virt/coco/arm-cca-host/arm-cca.c | 10 ++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 61 +++++++++++++++++++++++-
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  2 +
 5 files changed, 95 insertions(+), 2 deletions(-)
arch/arm64/include/asm/rmi_cmds.h +10 -0
index 25197f47a0a9..eb4f67eb6b01 100644
@@ -609,4 +609,25 @@ static inline unsigned long rmi_vdev_get_state(unsigned long vdev_phys, unsigned
 	return res.a0;
 }
 
+static inline unsigned long rmi_vdev_stop(unsigned long vdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_STOP, vdev_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_vdev_destroy(unsigned long rd,

arch/arm64/include/asm/rmi_cmds.h static inline unsigned long rmi_vdev_get_state(unsigned long vdev_phys, unsigned · line 631

Quoted 10 lines
arch/arm64/include/asm/rmi_cmds.h +10 -0
+					     unsigned long pdev_phys,
+					     unsigned long vdev_phys)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_VDEV_DESTROY, rd, pdev_phys, vdev_phys, &res);
+
+	return res.a0;
+}
+

One is enough.

arch/arm64/include/asm/rmi_smc.h line 59

Quoted 13 lines
arch/arm64/include/asm/rmi_cmds.h +1 -0
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +1 -1
index 127dd0938604..c6e16ab608e1 100644
@@ -55,8 +55,9 @@
 #define SMC_RMI_PDEV_STOP		SMC_RMI_CALL(0x017c)
 #define SMC_RMI_VDEV_COMMUNICATE	SMC_RMI_CALL(0x0186)
 #define SMC_RMI_VDEV_CREATE		SMC_RMI_CALL(0x0187)
+#define SMC_RMI_VDEV_DESTROY		SMC_RMI_CALL(0x0188)
 #define SMC_RMI_VDEV_GET_STATE		SMC_RMI_CALL(0x0189)
-

There have been a few of these. Check v2 carefully to make sure no more sneak in.

Quoted 27 lines
arch/arm64/include/asm/rmi_smc.h +1 -0
+#define SMC_RMI_VDEV_STOP		SMC_RMI_CALL(0x018A)
 
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0

static void cca_tsm_remove(void *tsm_core)

drivers/virt/coco/arm-cca-host/rmm-da.c +16 -0
index 8635f361bbe8..53072610fa67 100644
 
+void rme_unbind_vdev(struct realm *realm, struct pci_dev *pdev, struct pci_dev *pf0_dev)
+{
+	int ret;
+	phys_addr_t rmm_pdev_phys;
+	phys_addr_t rmm_vdev_phys;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct cca_host_tdi *host_tdi;
+	phys_addr_t rd_phys = virt_to_phys(realm->rd);
+
+	host_tdi = container_of(pdev->tsm->tdi, struct cca_host_tdi, tdi);
+	rmm_vdev_phys = virt_to_phys(host_tdi->rmm_vdev);
+
+	dsc_pf0 = to_cca_dsc_pf0(pf0_dev);
+	rmm_pdev_phys = virt_to_phys(dsc_pf0->rmm_pdev);
+	/* Request stopping the VDEV */
+	ret = rmi_vdev_stop(rmm_vdev_phys);

drivers/virt/coco/arm-cca-host/rmm-da.c void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev, · line 660

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+	if (ret) {
+		pr_err("failed to stop vdev (%d)\n", ret);
+		return;
+	}
+
+	schedule_vdev_unbind(pdev);
+	ret = rmi_vdev_destroy(rd_phys, rmm_pdev_phys, rmm_vdev_phys);
+	if (ret) {
+		pr_err("failed to destroy vdev (%d)\n", ret);
+		return;

No point in returning here. Maybe fine to keep this if more code is coming after this in future patches.

Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +2 -0
+	}
+}
#127 lore raw

Re: [RFC PATCH v1 23/38] coco: guest: arm64: Update arm CCA guest driver

In reply to Aneesh Kumar K.V (Arm)
Quoted 11 lines · On Mon, 28 Jul 2025 19:22:00 +0530 "Aneesh Kumar K.V (Arm)"…

This patch includes renaming changes to simplify the registration of a TSM backend in the next patch. There are no functional changes in this update.

index 0c9ea24a200c..547fc2c79f7d 100644

drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1 -0
 
+static int cca_guest_probe(struct platform_device *pdev)

...

drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 208

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +2 -7
+	ret = devm_add_action_or_reset(&pdev->dev, unregister_cca_tsm_report, NULL);
 
-/**
- * arm_cca_guest_exit - unregister with the Trusted Security Module (TSM)
- * interface.
- */
-static void __exit arm_cca_guest_exit(void)
-{
-	tsm_report_unregister(&arm_cca_tsm_ops);
+	return ret;

return devm_add_action_or_reset()

Mind you, Jason probably won't like this ;)

drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 227

Quoted 6 lines

}

MODULE_AUTHOR("Sami Mujawar <[email protected]>");

drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1 -1
-MODULE_DESCRIPTION("Arm CCA Guest TSM Driver");
+MODULE_DESCRIPTION("Arm CCA Guest TSM driver");

Is this D/d worth the noise?

Quoted 1 line

MODULE_LICENSE("GPL");

#131 lore raw

Re: [RFC PATCH v1 24/38] arm64: CCA: Register guest tsm callback

In reply to Aneesh Kumar K.V (Arm)
Quoted 6 lines · On Mon, 28 Jul 2025 19:22:01 +0530 "Aneesh Kumar K.V (Arm)"…

Register the TSM callback if the DA feature is supported by RSI.

Additionally, adjust the build order so that the TSM class is created before the arm-cca-guest driver initialization.

See below.

Quoted 13 lines
arch/arm64/kernel/rsi.c +1 -0
index bf9ea99e2aa1..ef06c083990a 100644
@@ -15,6 +15,7 @@
 #include <asm/rsi.h>
 
 static struct realm_config config;
+static unsigned long rsi_feat_reg0;
 
 unsigned long prot_ns_shared;
 EXPORT_SYMBOL(prot_ns_shared);

arch/arm64/kernel/rsi.c EXPORT_SYMBOL(prot_ns_shared); · line 28

Quoted 6 lines
arch/arm64/kernel/rsi.c +3 -0
@@ -22,6 +23,12 @@ EXPORT_SYMBOL(prot_ns_shared);
 DEFINE_STATIC_KEY_FALSE_RO(rsi_present);
 EXPORT_SYMBOL(rsi_present);
 
+bool rsi_has_da_feature(void)
+{
+	return !!u64_get_bits(rsi_feat_reg0, RSI_FEATURE_REGISTER_0_DA);

!! not needed.

Quoted 22 lines
arch/arm64/kernel/rsi.c +2 -0
+}
+EXPORT_SYMBOL_GPL(rsi_has_da_feature);
 
 
drivers/virt/coco/arm-cca-guest/arm-cca.c +3 -1
index 547fc2c79f7d..3adbbd67e06e 100644
@@ -1,6 +1,6 @@
 
 static int cca_guest_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -200,11 +256,22 @@ static int cca_guest_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	ret = tsm_report_register(&arm_cca_tsm_report_ops, NULL);
-	if (ret < 0)
+	if (ret < 0) {
 		pr_err("Error %d registering with TSM\n", ret);
+		goto err_out;
+	}

drivers/virt/coco/arm-cca-guest/arm-cca.c static int cca_guest_probe(struct platform_device *pdev) · line 272

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +8 -0
 
 	ret = devm_add_action_or_reset(&pdev->dev, unregister_cca_tsm_report, NULL);
+	if (ret < 0) {
+		pr_err("Error %d registering devm action\n", ret);
+		unregister_cca_tsm_report(NULL);
+		goto err_out;
+	}
+
+	if (rsi_has_da_feature())
+		ret = cca_tsm_register(pdev);

Why do we not need to call unregister_cca_tsm_report() if this fails?

drivers/virt/coco/arm-cca-guest/arm-cca.c static int cca_guest_probe(struct platform_device *pdev) · line 274

Quoted 2 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +1 -0
+err_out:

I'd just return above.

Quoted 11 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +0 -0
 	return ret;
 }
 
drivers/virt/coco/arm-cca-guest/rsi-da.h new file +2 -0
new file mode 100644
index 000000000000..8a4d5f1b0263
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*

drivers/virt/coco/arm-cca-guest/rsi-da.h line 12

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h new file +10 -0
+ * Copyright (C) 2025 ARM Ltd.
+ */
+
+#ifndef RSI_DA_H_
+#define RSI_DA_H_
+
+#include <linux/pci.h>
+#include <linux/pci-tsm.h>
+#include <asm/rsi_smc.h>
+

One blank line probably enough.

Quoted 15 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h new file +15 -0
+
+struct cca_guest_dsc {
+	struct pci_tsm_pf0 pci;
+};
+
+static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
+{
+	struct pci_tsm *tsm = pdev->tsm;
+
+	if (!tsm)
+		return NULL;
+	return container_of(tsm, struct cca_guest_dsc, pci.tsm);
+}
+
+#endif
#134 lore raw

Re: [RFC PATCH v1 25/38] cca: guest: arm64: Realm device lock support

In reply to Aneesh Kumar K.V (Arm)
Quoted 18 lines · On Mon, 28 Jul 2025 19:22:02 +0530 "Aneesh Kumar K.V (Arm)"…

Writing 1 to 'tsm/lock' will initiate the TDISP lock sequence.

---
 arch/arm64/include/asm/rsi_cmds.h         | 32 +++++++++++++-
 arch/arm64/include/asm/rsi_smc.h          |  5 +++
 drivers/virt/coco/arm-cca-guest/Makefile  |  2 +-
 drivers/virt/coco/arm-cca-guest/arm-cca.c | 18 ++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.c  | 52 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-guest/rsi-da.h  |  3 +-
 6 files changed, 108 insertions(+), 4 deletions(-)
 create mode 100644 drivers/virt/coco/arm-cca-guest/rsi-da.c
arch/arm64/include/asm/rsi_cmds.h +0 -0
index d4834baeef1b..b9c4b8ff5631 100644

arch/arm64/include/asm/rsi_cmds.h static inline int rsi_features(unsigned long index, unsigned long *out) · line 176

Quoted 7 lines
arch/arm64/include/asm/rsi_cmds.h +2 -2
@@ -172,8 +172,36 @@ static inline int rsi_features(unsigned long index, unsigned long *out)
 
 	arm_smccc_1_1_invoke(SMC_RSI_FEATURES, index, &res);
 
-	if (out)
-		*out = res.a1;
+	*out = res.a1;
+	return res.a0;

Seems unrelated change.

Quoted 20 lines
arch/arm64/include/asm/rsi_cmds.h +2 -0
+}
+
 
drivers/virt/coco/arm-cca-guest/rsi-da.c new file +10 -0
new file mode 100644
index 000000000000..097cf52ee199
@@ -0,0 +1,52 @@
 
+int rsi_device_lock(struct pci_dev *pdev)
+{
+	unsigned long ret;
+	struct cca_guest_dsc *dsm = to_cca_guest_dsc(pdev);
+	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
+		PCI_DEVID(pdev->bus->number, pdev->devfn);
+
+	ret = rsi_rdev_get_instance_id(vdev_id, &dsm->instance_id);
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to get the device instance id (%lu)\n", ret);

drivers/virt/coco/arm-cca-guest/rsi-da.c line 51

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c new file +10 -0
+		return -EIO;
+	}
+
+	ret = rsi_rdev_lock(pdev, vdev_id, dsm->instance_id);
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to lock the device (%lu)\n", ret);
+		return -EIO;
+	}
+
+	return ret;

return 0?

You carefully overwrite other error codes. I assume RSI_SUCCESS == 0 but even better to just return 0 directly in the good path.

drivers/virt/coco/arm-cca-guest/rsi-da.h line 13

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c new file +1 -0
+}
drivers/virt/coco/arm-cca-guest/rsi-da.h +0 -1
index 8a4d5f1b0263..f12430c7d792 100644
@@ -10,9 +10,9 @@
 #include <linux/pci-tsm.h>
 #include <asm/rsi_smc.h>
 
-

Push back to earlier patch where I comment on this.

Quoted 12 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +2 -0
 struct cca_guest_dsc {
 	struct pci_tsm_pf0 pci;
+	unsigned long instance_id;
 };
 
 static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
@@ -24,4 +24,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
 	return container_of(tsm, struct cca_guest_dsc, pci.tsm);
 }
 
+int rsi_device_lock(struct pci_dev *pdev);
 #endif
#136 lore raw

Re: [RFC PATCH v1 26/38] KVM: arm64: Add exit handler related to device assignment

In reply to Aneesh Kumar K.V (Arm)
Quoted 19 lines · On Mon, 28 Jul 2025 19:22:03 +0530 "Aneesh Kumar K.V (Arm)"…

Different RSI calls related to DA result in REC exits. Add a facility to register handlers for handling these REC exits.

arch/arm64/include/asm/rmi_smc.h +0 -0
index c6e16ab608e1..a5ef68b62bc0 100644
arch/arm64/kvm/rme-exit.c +2 -0
index 1a8ca7526863..25948207fc5b 100644
 
 
+static int rec_exit_dev_mem_map(struct kvm_vcpu *vcpu)
+{

arch/arm64/kvm/rme-exit.c static int rec_exit_host_call(struct kvm_vcpu *vcpu) · line 181

Quoted 10 lines
arch/arm64/kvm/rme-exit.c +10 -0
+	int ret;
+	struct realm_rec *rec = &vcpu->arch.rec;
+
+	if (realm_exit_dev_mem_map_handler) {
+		ret = (*realm_exit_dev_mem_map_handler)(rec);
+	} else {
+		kvm_pr_unimpl("Unsupported exit reason: %u\n",
+			      rec->run->exit.exit_reason);
+		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		ret = 0;

I guess maybe this gets more complex later, but right now

		return 0;

And maybe return in the if branch as well.
Same for other similar code in this patch.

Quoted 20 lines
arch/arm64/kvm/rme-exit.c +10 -0
+	}
+	return ret;
+}
+
 static void update_arch_timer_irq_lines(struct kvm_vcpu *vcpu)
 {
 	struct realm_rec *rec = &vcpu->arch.rec;
@@ -198,6 +252,12 @@ int handle_rec_exit(struct kvm_vcpu *vcpu, int rec_run_ret)
 		return rec_exit_ripas_change(vcpu);
 	case RMI_EXIT_HOST_CALL:
 		return rec_exit_host_call(vcpu);
+	case RMI_EXIT_VDEV_REQUEST:
+		return rec_exit_vdev_request(vcpu);
+	case RMI_EXIT_VDEV_COMM:
+		return rec_exit_vdev_communication(vcpu);
+	case RMI_EXIT_DEV_MEM_MAP:
+		return rec_exit_dev_mem_map(vcpu);
 	}
 
 	kvm_pr_unimpl("Unsupported exit reason: %u\n",
#141 lore raw

Re: [RFC PATCH v1 30/38] coco: host: arm64: Add support for realm host interface (RHI)

In reply to Aneesh Kumar K.V (Arm)
Quoted 4 lines · On Mon, 28 Jul 2025 19:22:07 +0530 "Aneesh Kumar K.V (Arm)"…

Device assignment-related RHI calls result in a REC exit, which is handled by the tsm guest_request callback.

Comments below.

Quoted 28 lines
---
 arch/arm64/include/asm/rhi.h             | 32 ++++++++++
 drivers/virt/coco/arm-cca-host/arm-cca.c | 68 ++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 81 ++++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  4 ++
 include/linux/tsm.h                      |  3 +
 5 files changed, 188 insertions(+)
 create mode 100644 arch/arm64/include/asm/rhi.h
drivers/virt/coco/arm-cca-host/arm-cca.c +14 -0
index be1296fb1bf2..0807fcf8d222 100644
 
+static int cca_tsm_guest_req(struct pci_dev *pdev, struct tsm_guest_req_info *info)
+{
+	int ret;
+
+	switch (info->type) {
+	case ARM_CCA_DA_OBJECT_SIZE:
+	{
+		int object_size;
+		struct da_object_size_req req;
+
+		if (sizeof(req) != info->req_len)
+			return -EINVAL;
+
+		if (copy_from_user(&req, info->req, info->req_len))

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 295

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10 -0
+			return -EFAULT;
+
+		object_size = rme_get_da_object_size(pdev, req.object_type);
+		if (object_size > 0) {
+			if (info->resp_len < sizeof(object_size))
+				return -EINVAL;
+			if (copy_to_user(info->resp, &object_size, sizeof(object_size)))
+				return -EFAULT;
+			info->resp_len = sizeof(object_size);
+			ret = 0;
			return 0;

then else isn't needed.

drivers/virt/coco/arm-cca-host/arm-cca.c

Quoted 1 line
drivers/virt/coco/arm-cca-host/arm-cca.c +1 -0
+		} else

#Style is {} on all legs if any need it.

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 298

Quoted 2 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +2 -0
+			/* error */
+			ret = object_size;
			return object_size;
Quoted 9 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +9 -0
+		break;
+	}
+	case ARM_CCA_DA_OBJECT_READ:
+	{
+		int resp_len;
+		struct da_object_read_req req;
+
+		if (sizeof(req) != info->req_len)
+			return -EINVAL;

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 317

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10 -0
+
+		if (copy_from_user(&req, info->req, info->req_len))
+			return -EFAULT;
+
+		resp_len = rme_da_object_read(pdev, req.object_type, req.offset,
+					      info->resp_len,
+					      info->resp);
+		if (resp_len > 0) {
+			info->resp_len = resp_len;
+			ret = 0;

Similar to above.

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_unbind(struct pci_tdi *tdi) · line 327

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10 -0
+		} else
+			/* error */
+			ret = resp_len;
+		break;
+	}
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;

As you've probably figured out, I love an early return. This sort of function shows why as it reduced indent etc in lots of places. Here you mix and match. Maybe it will make sense later in the series though!

Quoted 38 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +1 -0
+}
drivers/virt/coco/arm-cca-host/rmm-da.c +20 -0
index bef33e618fd3..c7da9d12f258 100644
@@ -10,7 +10,9 @@
 #include <keys/asymmetric-type.h>
 #include <keys/x509-parser.h>
 #include <linux/kvm_types.h>
+#include <linux/kvm_host.h>
 #include <asm/kvm_rme.h>
+#include <asm/kvm_emulate.h>
 
 #include "rmm-da.h"
 
@@ -769,6 +771,85 @@ static int rme_exit_vdev_req_handler(struct realm_rec *rec)
 	return 1;
 }
 
+int rme_get_da_object_size(struct pci_dev *pdev, int type)
+{
+	int ret = 0;
+	unsigned long len;
+	struct pci_tsm *tsm = pdev->tsm;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+
+	if (!tsm)
+		return -EINVAL;
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->dsm_dev);
+
+	/* Determine the buffer that should be used */
+	if (type == RHI_DA_OBJECT_INTERFACE_REPORT) {
+		len = dsc_pf0->interface_report.size;
+	} else if (type == RHI_DA_OBJECT_MEASUREMENT) {
+		len = dsc_pf0->measurements.size;
+	} else if (type == RHI_DA_OBJECT_CERTIFICATE) {

drivers/virt/coco/arm-cca-host/rmm-da.c static int rme_exit_vdev_req_handler(struct realm_rec *rec) · line 801

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+		len = dsc_pf0->cert_chain.cache.size;
+	} else if (type == RHI_DA_OBJECT_VCA) {
+		len = dsc_pf0->vca.size;
+	} else {
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	return len;
+err_out:

Similar to below. This pattern just makes things more complex. If we need to introduce a label, do it in the patch where you add code to do something on error.

Quoted 38 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +38 -0
+	return ret;
+}
+
+int rme_da_object_read(struct pci_dev *pdev, int type, unsigned long offset,
+		       unsigned long max_len, void __user *user_buf)
+{
+	void *buf;
+	int ret = 0;
+	unsigned long len;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct pci_tsm *tsm = pdev->tsm;
+
+	if (!tsm)
+		return -EINVAL;
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->dsm_dev);
+
+	/* Determine the buffer that should be used */
+	if (type == RHI_DA_OBJECT_INTERFACE_REPORT) {
+		len = dsc_pf0->interface_report.size;
+		buf = dsc_pf0->interface_report.buf;
+	} else if (type == RHI_DA_OBJECT_MEASUREMENT) {
+		len = dsc_pf0->measurements.size;
+		buf = dsc_pf0->measurements.buf;
+	} else if (type == RHI_DA_OBJECT_CERTIFICATE) {
+		len = dsc_pf0->cert_chain.cache.size;
+		buf = dsc_pf0->cert_chain.cache.buf;
+	} else if (type == RHI_DA_OBJECT_VCA) {
+		len = dsc_pf0->vca.size;
+		buf = dsc_pf0->vca.buf;
+	} else {
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	/* Assume that the buffer is large enough for the whole report */
+	if ((max_len - offset) < len) {
+		/* FIXME!! the error code */

drivers/virt/coco/arm-cca-host/rmm-da.c static int rme_exit_vdev_req_handler(struct realm_rec *rec) · line 849

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	if (copy_to_user(user_buf + offset, buf, len)) {
+		ret = -EIO;
+		goto err_out;
+	}
+	ret = len;
+err_out:

Definitely makes sense to just return directly in the error paths above and just return len here

Quoted 39 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +3 -0
+	return ret;
+}
+
 void rme_register_exit_handlers(void)
 {
 	realm_exit_vdev_comm_handler = rme_exit_vdev_comm_handler;
drivers/virt/coco/arm-cca-host/rmm-da.h +4 -0
index cebddab8464d..457660ff3b69 100644
@@ -10,6 +10,7 @@
 #include <linux/pci-ide.h>
 #include <linux/pci-tsm.h>
 #include <asm/rmi_smc.h>
+#include <asm/rhi.h>
 
 #define MAX_CACHE_OBJ_SIZE	4096
 struct cache_object {
@@ -101,4 +102,7 @@ void *rme_create_vdev(struct realm *realm, struct pci_dev *pdev,
 void rme_unbind_vdev(struct realm *realm, struct pci_dev *pdev,
 		     struct pci_dev *pf0_dev);
 void rme_register_exit_handlers(void);
+int rme_get_da_object_size(struct pci_dev *pdev, int type);
+int rme_da_object_read(struct pci_dev *pdev, int type, unsigned long offset,
+		       unsigned long max_len, void __user *user_buf);
 #endif
include/linux/tsm.h +3 -0
index 497a3b4df5a0..e82046b0c7fa 100644
@@ -145,5 +145,8 @@ struct tsm_guest_req_info {
 	size_t resp_len;
 };
 
+#define ARM_CCA_DA_OBJECT_SIZE 0x1
+#define ARM_CCA_DA_OBJECT_READ 0x2
+
 int tsm_guest_req(struct device *dev, struct tsm_guest_req_info *info);
 #endif /* __TSM_H */
#143 lore raw

Re: [RFC PATCH v1 31/38] coco: guest: arm64: Add support for fetching interface report and certificate chain from host

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:22:08 +0530 "Aneesh Kumar K.V (Arm)"…

Fetch interface report and certificate chain from the host using RHI calls.

Comments inline

Quoted 24 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +9 -0
index 28ec946df1e2..47b379318e7c 100644
@@ -4,6 +4,7 @@
  */
 
 #include <linux/pci.h>
+#include <linux/mem_encrypt.h>
 #include <asm/rsi_cmds.h>
 
 #include "rsi-da.h"
@@ -50,6 +51,121 @@ rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id,
 	return RSI_SUCCESS;
 }
 
+static long rhi_get_report(int vdev_id, int da_object_type, void **report, int *report_size)
+{
+	int ret, enc_ret = 0;
+	int nr_pages;
+	int max_data_len;
+	void *data_buf_shared, *data_buf_private;
+	struct rsi_host_call *rhicall;
+

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id, · line 71

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+	rhicall = kmalloc(sizeof(struct rsi_host_call), GFP_KERNEL);
+	if (!rhicall)
+		return -ENOMEM;
+
+	rhicall->imm = 0;
+	rhicall->gprs[0] = RHI_DA_FEATURES;
+
+	ret = rsi_host_call(virt_to_phys(rhicall));
+	if (ret != RSI_SUCCESS) {
+		ret =  -EIO;

Extra space.

Quoted 34 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +34 -0
+		goto err_out;
+	}
+
+	if (rhicall->gprs[0] != 0x3) {
+		ret =  -EIO;
+		goto err_out;
+	}
+
+	rhicall->imm = 0;
+	rhicall->gprs[0] = RHI_DA_OBJECT_SIZE;
+	rhicall->gprs[1] = vdev_id;
+	rhicall->gprs[2] = da_object_type;
+
+	ret = rsi_host_call(virt_to_phys(rhicall));
+	if (ret != RSI_SUCCESS) {
+		ret =  -EIO;
+		goto err_out;
+	}
+	if (rhicall->gprs[0] != RHI_DA_SUCCESS) {
+		ret =  -EIO;
+		goto err_out;
+	}
+	max_data_len = rhicall->gprs[1];
+	*report_size = max_data_len;
+
+	/*
+	 * We need to share this memory with hypervisor.
+	 * So it should be multiple of sharing unit.
+	 */
+	max_data_len = ALIGN(max_data_len, PAGE_SIZE);
+	nr_pages = max_data_len >> PAGE_SHIFT;
+
+	if (!max_data_len || nr_pages > MAX_ORDER_NR_PAGES) {
+		ret = -ENOMEM;

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id, · line 115

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+		goto err_out;
+	}
+
+	/*
+	 * We need to share this memory with hypervisor.
+	 * So it should be multiple of sharing unit.
+	 */
+	data_buf_shared = (void *)__get_free_pages(GFP_KERNEL, get_order(max_data_len));
+	if (!data_buf_shared) {
+		ret =  -ENOMEM;

extra space. All of these seem to have one. Not seeing a reason for it though.

Quoted 37 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +37 -0
+		goto err_out;
+	}
+
+	data_buf_private = kmalloc(*report_size, GFP_KERNEL);
+	if (!data_buf_private) {
+		ret =  -ENOMEM;
+		goto err_private_alloc;
+	}
+
+	ret = set_memory_decrypted((unsigned long)data_buf_shared, nr_pages);
+	if (ret) {
+		ret =  -EIO;
+		goto err_decrypt;
+	}
+
+	rhicall->imm = 0;
+	rhicall->gprs[0] = RHI_DA_OBJECT_READ;
+	rhicall->gprs[1] = vdev_id;
+	rhicall->gprs[2] = da_object_type;
+	rhicall->gprs[3] = 0; /* offset within the data buffer */
+	rhicall->gprs[4] = max_data_len;
+	rhicall->gprs[5] = virt_to_phys(data_buf_shared);
+	ret = rsi_host_call(virt_to_phys(rhicall));
+	if (ret != RSI_SUCCESS || rhicall->gprs[0] != RHI_DA_SUCCESS) {
+		ret =  -EIO;
+		goto err_rhi_call;
+	}
+
+	memcpy(data_buf_private, data_buf_shared, *report_size);
+	enc_ret = set_memory_encrypted((unsigned long)data_buf_shared, nr_pages);
+	if (!enc_ret)
+		/* If we fail to mark it encrypted don't free it back */
+		free_pages((unsigned long)data_buf_shared, get_order(max_data_len));
+
+	*report = data_buf_private;
+	kfree(rhicall);
+	return 0;

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_interface_report(struct pci_dev *pdev, unsigned long vdev_id, · line 162

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+
+err_rhi_call:
+	enc_ret = set_memory_encrypted((unsigned long)data_buf_shared, nr_pages);
+err_decrypt:
+	kfree(data_buf_private);
+err_private_alloc:
+	if (!enc_ret)
+		/* If we fail to mark it encrypted don't free it back */
+		free_pages((unsigned long)data_buf_shared, get_order(max_data_len));
+err_out:

I'd expect there to be nothing to do except return under an err_out label So rename it.

Quoted 20 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +13 -0
+	*report = NULL;
+	*report_size = 0;
+	kfree(rhicall);
+	return ret;
+}
+
 int rsi_device_lock(struct pci_dev *pdev)
 {
 	unsigned long ret;
@@ -82,5 +198,20 @@ int rsi_device_lock(struct pci_dev *pdev)
 		return -EOPNOTSUPP;
 	}
 
+	/* Now make a host call to copy the interface report to guest. */
+	ret = rhi_get_report(vdev_id, RHI_DA_OBJECT_INTERFACE_REPORT,
+			     &dsm->interface_report, &dsm->interface_report_size);
+	if (ret) {
+		pci_err(pdev, "failed to get interface report from the host (%lu)\n", ret);
+		return -EIO;
+	}

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 217

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +8 -0
+
+	ret = rhi_get_report(vdev_id, RHI_DA_OBJECT_CERTIFICATE,
+			     &dsm->certificate, &dsm->certificate_size);
+	if (ret) {
+		pci_err(pdev, "failed to get device certificate from the host (%lu)\n", ret);
+		return -EIO;
+	}
+
 
 	return ret;

return 0;

Quoted 1 line

}

#145 lore raw

Re: [RFC PATCH v1 32/38] coco: guest: arm64: Add support for guest initiated TDI bind/unbind

In reply to Aneesh Kumar K.V (Arm)
Quoted 6 lines · On Mon, 28 Jul 2025 19:22:09 +0530 "Aneesh Kumar K.V (Arm)"…

Add RHI for VDEV_SET_TDI_STATE

Note: This is not part of RHI spec. This is a POC implementation and will be later switced to correct interface defined by RHI.

A few minor comments.

Maybe we need some discussion of how this is used.

Quoted 14 lines
arch/arm64/kernel/rhi.c new file +8 -0
new file mode 100644
index 000000000000..3685b50c2e94
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 ARM Ltd.
+ */
+
+#include <asm/memory.h>
+#include <asm/string.h>
+#include <asm/rsi.h>

arch/arm64/kernel/rhi.c line 18

Quoted 10 lines
arch/arm64/kernel/rhi.c new file +10 -0
+#include <asm/rhi.h>
+
+#include <linux/slab.h>
+
+long rhi_da_vdev_set_tdi_state(unsigned long guest_rid, unsigned long target_state)
+{
+	long ret;
+	struct rsi_host_call *rhi_call;
+
+	rhi_call = kmalloc(sizeof(struct rsi_host_call), GFP_KERNEL);
	struct rsi_host_call *rhi_call __free(kfree) =
		kmalloc(sizeof(*rhi_call), GFP_KERNEL);

then direct returns on errors.

Quoted 22 lines
arch/arm64/kernel/rhi.c new file +17 -0
+	if (!rhi_call)
+		return -ENOMEM;
+
+	rhi_call->imm = 0;
+	rhi_call->gprs[0] = RHI_DA_VDEV_SET_TDI_STATE;
+	rhi_call->gprs[1] = guest_rid;
+	rhi_call->gprs[2] = target_state;
+
+	ret = rsi_host_call(virt_to_phys(rhi_call));
+	if (ret != RSI_SUCCESS)
+		ret =  -EIO;
+	else
+		ret = rhi_call->gprs[0];
+
+	kfree(rhi_call);
+	return ret;
+}
drivers/virt/coco/arm-cca-guest/arm-cca.c +0 -0
index 2c0190bcb2a9..de70fba09e92 100644

drivers/virt/coco/arm-cca-guest/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 225

Quoted 5 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +1 -1
@@ -222,11 +222,20 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)
 
 static int cca_tsm_lock(struct pci_dev *pdev)
 {
-	unsigned long ret;
+	long ret;

Push this earlier. It doesn't do any harm that I can see and will reduce churn

Quoted 27 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +10 -1
+	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
+		PCI_DEVID(pdev->bus->number, pdev->devfn);
 
+	ret = rhi_da_vdev_set_tdi_state(vdev_id, RHI_DA_TDI_CONFIG_LOCKED);
+	if (ret) {
+		pci_err(pdev, "failed to TSM bind the device (%ld)\n", ret);
+		return -EIO;
+	}
+
+	/* This will be done by above rhi in later spec */
 	ret = rsi_device_lock(pdev);
 	if (ret) {
-		pci_err(pdev, "failed to lock the device (%lu)\n", ret);
+		pci_err(pdev, "failed to lock the device (%ld)\n", ret);
 		return -EIO;
 	}
 	return 0;

static const struct pci_tsm_ops cca_pci_ops = {

drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
index 0807fcf8d222..18d0a627baa4 100644
@@ -254,9 +254,13 @@ static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_de
 static void cca_tsm_unbind(struct pci_tdi *tdi)
 {

drivers/virt/coco/arm-cca-host/arm-cca.c static struct pci_tdi *cca_tsm_bind(struct pci_dev *pdev, struct pci_dev *pf0_de · line 259

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +6 -2
 	struct realm *realm = &tdi->kvm->arch.realm;
-
+	/*
+	 * FIXME!!
+	 * All the related DEV RIPAS regions should be unmapped by now.
+	 * For now we handle them during stage2 teardown. There is no
+	 * bound IPA address available here. Possibly dmabuf can help
+	 */
 	rme_unbind_vdev(realm, tdi->pdev, tdi->pdev->tsm->dsm_dev);
-

Tidy up these whitespace changes. Just adds noise.

Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
 	module_put(THIS_MODULE);
 }
 
#149 lore raw

Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report

In reply to Aneesh Kumar K.V (Arm)
Quoted 4 lines · On Mon, 28 Jul 2025 19:22:11 +0530 "Aneesh Kumar K.V (Arm)"…

This starts the sequence to transition the realm device to the TDISP RUN state by writing 1 to 'tsm/accept'.

Just some trivial stuff.

Quoted 15 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
index 47b379318e7c..936f844880de 100644
@@ -215,3 +215,135 @@ int rsi_device_lock(struct pci_dev *pdev)
 
 	return ret;
 }
 
+
+int rsi_update_interface_report(struct pci_dev *pdev, bool validate)
+{
+	int ret;
+	struct resource *r;
+	int msix_tbl_bar, msix_pba_bar;

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 274

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+	unsigned int range_id;
+	unsigned long mmio_start_phys;
+	unsigned long mmio_flags = 0; /* non coherent, not limited order */
+	struct pci_tdisp_mmio_range *mmio_range;
+	struct pci_tdisp_device_interface_report *interface_report;
+	struct cca_guest_dsc *dsm = to_cca_guest_dsc(pdev);
+	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
+		PCI_DEVID(pdev->bus->number, pdev->devfn);
+
+

Single line.

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 278

Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4 -0
+	interface_report = (struct pci_tdisp_device_interface_report *)dsm->interface_report;
+	mmio_range = (struct pci_tdisp_mmio_range *)(interface_report + 1);
+
+

Single line

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 288

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+	msix_tbl_bar = get_msix_bar(pdev, PCI_MSIX_TABLE);
+	msix_pba_bar = get_msix_bar(pdev, PCI_MSIX_PBA);
+
+	for (int i = 0; i < interface_report->mmio_range_count; i++, mmio_range++) {
+
+		/*FIXME!! units in 4K size*/
+		range_id = FIELD_GET(TSM_INTF_REPORT_MMIO_RANGE_ID, mmio_range->range_attributes);
+
+		/* no secure interrupts */
+		if (msix_tbl_bar != -1 && range_id == msix_tbl_bar) {

That first condition can get hiked out of the loop.

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 293

Quoted 5 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +5 -0
+			pr_info("Skipping misx table\n");
+			continue;
+		}
+
+		if (msix_pba_bar != -1 && range_id == msix_pba_bar) {

Likewise.

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 301

Quoted 8 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +8 -0
+			pr_info("Skipping misx pba\n");
+			continue;
+		}
+
+		r = pci_resource_n(pdev, range_id);
+
+		if (r->end == r->start ||
+		    ((r->end - r->start + 1) & ~PAGE_MASK) || !mmio_range->num_pages) {

resource_size()

Quoted 35 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +35 -0
+			pci_warn(pdev, "Skipping broken range [%d] #%d %d pages, %llx..%llx\n",
+				i, range_id, mmio_range->num_pages, r->start, r->end);
+			continue;
+		}
+
+		if (FIELD_GET(TSM_INTF_REPORT_MMIO_IS_NON_TEE, mmio_range->range_attributes)) {
+			pci_info(pdev, "Skipping non-TEE range [%d] #%d %d pages, %llx..%llx\n",
+				 i, range_id, mmio_range->num_pages, r->start, r->end);
+			continue;
+		}
+
+		/* No secure interrupts, we should not find this set, ignore for now. */
+		if (FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes) ||
+		    FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes)) {
+			pci_info(pdev, "Skipping MSIX (%ld/%ld) range [%d] #%d %d pages, %llx..%llx\n",
+				 FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes),
+				 FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes),
+				 i, range_id, mmio_range->num_pages, r->start, r->end);
+			continue;
+		}
+
+		mmio_start_phys = mmio_range->first_page;
+		if (validate)
+			ret = rsi_validate_dev_mapping(vdev_id, dsm->instance_id,
+						       r->start, r->end + 1,
+						       mmio_start_phys << 12, mmio_flags);
+		else
+			ret = rsi_invalidate_dev_mapping(r->start, r->end + 1);
+		if (ret) {
+			pci_err(pdev, "failed to set protection attributes for the address range\n");
+			return -EIO;
+		}
+	}
+	return 0;
+}
#174 lore raw
In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · Aneesh Kumar K.V (Arm)

This patch series implements support for Device Assignment in the ARM CCA architecture. The code changes are based on Alp12 specification published here [1].

Robin and I were talking about CCA and DMA here:

https://lore.kernel.org/r/[email protected]

What do you think about pulling some of this out and trying to independently push a series getting the DMA API layers ready for device assignment?

I think there will be some discussion on these points, it would be good to get started.

Jason

#167 lore raw

Re: [RFC PATCH v1 37/38] coco: guest: arm64: Add support for fetching device measurements

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:22:14 +0530 "Aneesh Kumar K.V (Arm)"…

Fetch device measurements using RSI_RDEV_GET_MEASUREMENTS.

One completely trivial comment.

J

Quoted 7 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +0 -0
index 64034d220e02..6222b10964ee 100644
@@ -213,6 +245,13 @@ int rsi_device_lock(struct pci_dev *pdev)
 		return -EIO;

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 255

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +7 -0
 	}
 
+	ret = rhi_get_report(vdev_id, RHI_DA_OBJECT_MEASUREMENT,
+			     &dsm->measurements, &dsm->measurements_size);
+	if (ret) {
+		pci_err(pdev, "failed to get device certificate from the host (%lu)\n", ret);
+		return -EIO;
+	}
+
 	return ret;

return 0; Always good to make it explicit when it can't take any other values. Looks like that belong sin an earlier patch though based on this snippet.

Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.c +0 -0
 }
#170 lore raw

Re: [RFC PATCH v1 38/38] coco: guest: arm64: Add support for fetching device info

In reply to Aneesh Kumar K.V (Arm)
Quoted 5 lines · On Mon, 28 Jul 2025 19:22:15 +0530 "Aneesh Kumar K.V (Arm)"…

RSI_RDEV_GET_INFO returns different digest hash values, which can be compared with host cached values to ensure the host didn't tamper with the cached data.

Hi Aneesh

A few comments on this one

Jonathan

Quoted 41 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +26 -0
index 6222b10964ee..a1bb225adb4c 100644
@@ -6,6 +6,7 @@
 #include <linux/pci.h>
 #include <linux/mem_encrypt.h>
 #include <asm/rsi_cmds.h>
+#include <crypto/hash.h>
 
 #include "rsi-da.h"
 
@@ -186,11 +187,102 @@ rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id,
 	return RSI_SUCCESS;
 }
 
+static int verify_digests(struct cca_guest_dsc *dsm)
+{
+	int i;
+	int ret;
+	u8 digest[SHA512_DIGEST_SIZE];
+	int sdesc_size;
+	size_t digest_size;
+	char *hash_alg_name;
+	struct shash_desc *shash;
+	struct crypto_shash *alg;
+	struct pci_dev *pdev = dsm->pci.tsm.pdev;
+	struct {
+		uint8_t *report;
+		size_t size;
+		uint8_t *digest;
+	} reports[] = {
+		{
+			dsm->interface_report,
+			dsm->interface_report_size,
+			dsm->dev_info.report_digest
+		},
+		{
+			dsm->certificate,
+			dsm->certificate_size,
+			dsm->dev_info.cert_digest

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 224

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+		},
+		{
+			dsm->measurements,
+			dsm->measurements_size,
+			dsm->dev_info.meas_digest
+		}
+	};
+
+
+	if (dsm->dev_info.hash_algo == RSI_HASH_SHA_256) {

This to me smells like a place that will need switch sooner or later, maybe just do it now.

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 232

Quoted 8 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +8 -0
+		hash_alg_name = "sha256";
+		digest_size = SHA256_DIGEST_SIZE;
+	} else if (dsm->dev_info.hash_algo == RSI_HASH_SHA_512) {
+		hash_alg_name = "sha512";
+		digest_size = SHA512_DIGEST_SIZE;
+	} else {
+		pci_err(pdev, "unknown realm hash algorithm!\n");
+		ret = -EINVAL;

return -EINVAL;

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 236

Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4 -0
+		goto err_out;
+	}
+
+	alg = crypto_alloc_shash(hash_alg_name, 0, 0);

As below - I'd spin a DEFINE_FREE() for this to simplify error paths in here and remove the labels that had me confused briefly.

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 242

Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+	if (IS_ERR(alg)) {
+		pci_err(pdev, "cannot allocate %s\n", hash_alg_name);
+		return PTR_ERR(alg);
+	}
+
+	sdesc_size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);

Not common in crypto so perhaps just leave this as you have it.

	sdesc_size = struct_size(struct shash_dec, __ctx, crypto_shash_desc_size(alg));

is more informative on what is going on here.

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 253

Quoted 11 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +11 -0
+	shash = kzalloc(sdesc_size, GFP_KERNEL);
+	if (!shash) {
+		pci_err(pdev, "cannot allocate sdesc\n");
+		ret = -ENOMEM;
+		goto err_free_shash;
+	}
+	shash->tfm = alg;
+
+	for (i = 0; i < ARRAY_SIZE(reports); i++) {
+		ret = crypto_shash_digest(shash, reports[i].report,
+					  reports[i].size, digest);

To me a bit marginal on whether this loop and the structures above are beneficial over straight line code.

Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+		if (ret) {
+			pci_err(pdev, "failed to compute digest, %d\n", ret);
+			goto err_free_sdesc;
+		}
+
+		if (memcmp(reports[i].digest, digest, digest_size)) {

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 269

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+			pci_err(pdev, "invalid digest\n");
+			ret = -EINVAL;
+			goto err_free_sdesc;
+		}
+	}
+
+	kfree(shash);
+	crypto_free_shash(alg);
+
+	pci_info(pdev, "Successfully verified the digests\n");

debug.

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 272

Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +3 -0
+	return 0;
+
+err_free_sdesc:

I'd tweak these labels if you keep them. This isn't freeing the sdesc.

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 273

Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.c +1 -0
+	kfree(shash);

Looks perfect for some __free() magic dust.

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 275

Quoted 2 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +2 -0
+err_free_shash:
+	crypto_free_shash(alg);

DEFINE_FREE() needed for this but looks pretty uncontroversial.

drivers/virt/coco/arm-cca-guest/rsi-da.c rsi_rdev_get_measurements(struct pci_dev *pdev, unsigned long vdev_id, · line 276

Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.c +1 -0
+err_out:

As below. I'd not do this.

Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4 -0
+	return ret;
+}
+
 int rsi_device_lock(struct pci_dev *pdev)
 {
 	unsigned long ret;
 	unsigned long tdisp_version;
 	struct rsi_device_measurements_params *rsi_dev_meas;
+	struct rsi_device_info *dev_info;
 	struct cca_guest_dsc *dsm = to_cca_guest_dsc(pdev);
 	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
 		PCI_DEVID(pdev->bus->number, pdev->devfn);

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 348

Quoted 5 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +2 -0
@@ -252,8 +344,44 @@ int rsi_device_lock(struct pci_dev *pdev)
 		return -EIO;
 	}
 
+	/* RMM expects sizeof(dev_info) (512 bytes) aligned address */
+	dev_info = kmalloc(sizeof(*dev_info), GFP_KERNEL);

Use a __free(kfree) here (and direct returns on errors) given it's freed in all paths and we don't care if it is freed before or after verifying the digests.

I'm being slow today, but what in that enforces the alignment? I guess it's that the structure happens to be big enough that it happens naturally?

I'd allocate max(512, sizeof(*dev_info)) to make it explicitly the case.

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+	if (!dev_info) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	ret = rsi_rdev_get_info(vdev_id, dsm->instance_id, virt_to_phys(dev_info));
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to get device digests (%lu)\n", ret);
+		ret = -EIO;
+		kfree(dev_info);

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 368

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+		goto err_out;
+	}
+
+	dsm->dev_info.attest_type   = dev_info->attest_type;
+	dsm->dev_info.cert_id       = dev_info->cert_id;
+	dsm->dev_info.hash_algo     = dev_info->hash_algo;
+	memcpy(dsm->dev_info.cert_digest, dev_info->cert_digest, SHA512_DIGEST_SIZE);
+	memcpy(dsm->dev_info.meas_digest, dev_info->meas_digest, SHA512_DIGEST_SIZE);
+	memcpy(dsm->dev_info.report_digest, dev_info->report_digest, SHA512_DIGEST_SIZE);
+

Can't you memcpy the whole thing in one go?

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 381

Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +13 -0
+	kfree(dev_info);
+	/*
+	 * Verify that the digests of the provided reports match with the
+	 * digests from RMM
+	 */
+	ret = verify_digests(dsm);
+	if (ret) {
+		pci_err(pdev, "device digest validation failed (%ld)\n", ret);
+		return ret;
+	}
+
+	return 0;
+err_out:

I'll always grumble about these. To me it always makes the code less readable. Some others disagree though ;(

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 384

Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +1 -0
 	return ret;
 }
+

Looks like this should have been in an earlier patch.

Quoted 19 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +0 -0
 static inline unsigned long rsi_rdev_start(struct pci_dev *pdev,
 					   unsigned long vdev_id, unsigned long inst_id)
 {
drivers/virt/coco/arm-cca-guest/rsi-da.h +1 -0
index f26156d9be81..e8953b8e85a3 100644
@@ -10,6 +10,7 @@
 #include <linux/pci-tsm.h>
 #include <asm/rsi_smc.h>
 #include <asm/rhi.h>
+#include <crypto/sha2.h>
 
 struct pci_tdisp_device_interface_report {
 	u16 interface_info;
@@ -33,6 +34,17 @@ struct pci_tdisp_mmio_range {
 #define TSM_INTF_REPORT_MMIO_RESERVED		GENMASK(15, 4)
 #define TSM_INTF_REPORT_MMIO_RANGE_ID		GENMASK(31, 16)
 

drivers/virt/coco/arm-cca-guest/rsi-da.h struct pci_tdisp_mmio_range { · line 46

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +10 -0
+struct dsm_device_info {
+	u64 flags;
+	u64 attest_type;
+	u64 cert_id;
+	u64 hash_algo;
+	u8 cert_digest[SHA512_DIGEST_SIZE];
+	u8 meas_digest[SHA512_DIGEST_SIZE];
+	u8 report_digest[SHA512_DIGEST_SIZE];
+};
+

One probably enough.

Quoted 12 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +2 -0
+
 struct cca_guest_dsc {
 	struct pci_tsm_pf0 pci;
 	unsigned long instance_id;
@@ -42,6 +54,7 @@ struct cca_guest_dsc {
 	int certificate_size;
 	void *measurements;
 	int measurements_size;
+	struct dsm_device_info dev_info;
 };
 
 static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
#162 lore raw

Re: [RFC PATCH v1 35/38] coco: guest: arm64: Add Realm device start and stop support

In reply to Aneesh Kumar K.V (Arm)
Quoted 3 lines · On Mon, 28 Jul 2025 19:22:12 +0530 "Aneesh Kumar K.V (Arm)"…

Writing 1 to 'tsm/acceept' will initiate the TDISP RUN sequence.

A few more trivial things on this first read through.

Jonathan

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_start(struct pci_dev *pdev) · line 401

Quoted 11 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
index 936f844880de..64034d220e02 100644
 
+int rsi_device_stop(struct pci_dev *pdev)
+{
+	int ret;
+	struct cca_guest_dsc *dsm = to_cca_guest_dsc(pdev);
+	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
+		PCI_DEVID(pdev->bus->number, pdev->devfn);

Feels like this occurs so often we should add a helper. Can't be completely generic as pci_domain_nr can have more bits than I guess we are assuming here, but we can have something for use in the rsi code.

Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+
+	ret = rsi_rdev_stop(pdev, vdev_id, dsm->instance_id);
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to stop the device (%u)\n", ret);
+		return -EIO;
+	}
 	return 0;
 }
drivers/virt/coco/arm-cca-guest/rsi-da.h +0 -0
index 0d6e1c0ada4a..71ee1edb832e 100644

drivers/virt/coco/arm-cca-guest/rsi-da.h static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev) · line 57

Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +0 -1
@@ -54,5 +54,5 @@ static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)
 int rsi_update_interface_report(struct pci_dev *pdev, bool validate);
 int rsi_device_lock(struct pci_dev *pdev);
 int rsi_device_start(struct pci_dev *pdev);
-

Tidy this up.

Quoted 2 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +1 -0
+int rsi_device_stop(struct pci_dev *pdev);
 #endif
#150 lore raw

Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report

In reply to Aneesh Kumar K.V (Arm)
Quoted 6 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+	for (int i = 0; i < interface_report->mmio_range_count; i++, mmio_range++) {
+
+		/*FIXME!! units in 4K size*/
+		range_id = FIELD_GET(TSM_INTF_REPORT_MMIO_RANGE_ID, mmio_range->range_attributes);
+
+		/* no secure interrupts */

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 297

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+		if (msix_tbl_bar != -1 && range_id == msix_tbl_bar) {
+			pr_info("Skipping misx table\n");
+			continue;
+		}
+
+		if (msix_pba_bar != -1 && range_id == msix_pba_bar) {
+			pr_info("Skipping misx pba\n");
+			continue;
+		}
+

MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated. It is enough to have sufficient isolation within the BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.

Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?

- R2

#100 lore raw

Re: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm

In reply to Aneesh Kumar K.V (Arm)

drivers/virt/coco/arm-cca-host/rmm-da.c line 21

Quoted 10 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+static int pci_res_to_pdev_addr(struct rmi_pdev_addr_range *pdev_addr,
+				unsigned int naddr, struct resource *res,
+				unsigned int nres)
+{
+	int i, j;
+
+	for (i = 0, j = 0; i < naddr && j < nres; j++) {
+		if (res[j].flags & IORESOURCE_MEM) {
+			pdev_addr[i].base = res[j].start;
+			pdev_addr[i].top  = res[j].end;

I think there is an off-by-one bug in res[j].end. As per the RMM specification the base address is inclusive and the top address is exclusive. Both res[j].start and res[j].end are inclusive, and hence res[j].end seems wrong.

drivers/virt/coco/arm-cca-host/rmm-da.c line 59

Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2 -0
+	/* use the rid and MMIO resources of the epdev */
+	params->rid_top = params->rid_base = rid;

Similar issue here. As per the specification the rid_base is inclusive and the rid_top exclusive.

- R2

#84 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jonathan Cameron
Quoted 23 lines · Jonathan Cameron
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Is faux_device a better choice? I admit to not knowing entirely what it is for..

I'll go with a cautious yes to faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.

Not that many examples yet, but cpuidle-psci.c looks like a vaguely similar case to this one.

Quoted 7 lines

All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if probe() fails, which is cleaner in many ways than leaving a pointless stub behind.

Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.

Yeah, exactly. Can a TSM driver even be modular? If it has to be built in then there is no reason to do this:

Quoted 2 lines
Quoted 2 lines

The goal is to have tsm class device to be parented by the platform device.

IMHO the only real point of that is to trigger module autoloading.

Otherwise the tsm core should accept NULL as the parent pointer during registration, it probably already does..

Jason

#11 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Suzuki K Poulose
Quoted 8 lines · Suzuki K Poulose
Quoted 5 lines
Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

PCI SIG supports it, kernel should support it.

ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.

Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.

I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for the RMM S2?

The first option sounds fairly appealing, IMHO

Quoted 3 lines

Rather than mapping the entire memory from the host, it would be ideal if the Coco vms have some sort of a callback to "make sure the DMA wouldn't fault for a device".

Isn't that a different topic? For right now we expect that all pages are pinned and loaded into both S2s. Upon any private/shared conversion the pages should be reloaded into the appropriate S2s if required. The VM never needs to tell the hypervisor that it wants to do DMA.

There are all sorts of options here to relax this but exploring them it an entirely different project that CCA, IMHO.

Jason

#33 lore raw

Re: [RFC PATCH v1 06/38] iommufd: Add and option to request for bar mapping with IORESOURCE_EXCLUSIVE

In reply to Xu Yilun
Quoted 14 lines · Xu Yilun
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Aneesh Kumar K.V
Quoted 10 lines · Jason Gunthorpe
Quoted 1 line · Aneesh Kumar K.V (Arm)

Why would we need this?

I can sort of understand why Intel would need it due to their issues with MCE, but ARM shouldn't care either way, should it?

But also why is it an iommufd option? That doesn't seem right..

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Jason

This is based on our previous discussion https://lore.kernel.org/all/[email protected]

I suggested a global option, this is a per-device option, and that especially seems wrong for iommufd. If it is per-device that is vfio,

I think this should be per-device.

IMHO there is no use case for that, it should arguably be global to the whole kernel.

Quoted 2 lines

The original purpose of this pci_region_request_*() is to prevent further mmap/read/write against a vfio_cdev FD which would be used

No way, the VFIO internal mmap should be controled by VFIO not by request region. If you want to block that it should be blocked by iommufd telling VFIO that the device is bound which revokes the mmaps/dmabufs/etc and prevents opening new ones.

The only thing request region should do is prevent /sys/../resource, /dev/mem users and so on, which is why it can and should be global. Arguably VFIO should always block those things but historically hasn't..

There should only be one request region call in VFIO, it should ideally happen when the VFIO driver probes the device.

Jason

#94 lore raw

Re: [RFC PATCH v1 12/38] coco: host: arm64: CCA host platform device driver

In reply to Jonathan Cameron
Quoted 7 lines · Jonathan Cameron
Quoted 2 lines

devm is useful to solve complex things, these trivial things should be done normally..

Sure, that would be fine for now. If we end up with a large complex flow that happens to have a tsm_register() in amongst various managed resources we can revisit. If they all end up looking like this then a manual call in remove is fine.

IMHO just don't use devm, it is so easy to use devm wrong and get out of order clean up. It works well for extremely simple case where 100% of cleanup is in devm (but then it is questionable if the overhead is worthwehile), and it is necessary for extremely hard cases where writing a manual unwind is too hard. But the middle ground it tends to just make ordering bugs and not provide alot of value, IMHO.

Jason

#105 lore raw

Re: [RFC PATCH v1 14/38] coco: host: arm64: Device communication support

In reply to Jonathan Cameron
Quoted 9 lines · Jonathan Cameron

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 92

Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5 -0
+static int init_dev_communication_buffers(struct cca_host_comm_data *comm_data)
+{
+	int ret = -ENOMEM;
+
+	comm_data->io_params = (struct rmi_dev_comm_data *)get_zeroed_page(GFP_KERNEL);

Hmm. There isn't a DEFINE_FREE() yet for free_page(). Maybe time to add one. If we did then we'd use local variables until all allocations succeed then assign with no_free_ptr()

Maybe think carefully if you really need a "page".

What would prevent just using kzalloc(PAGE_SIZE)? Under the covers it is almost the same thing.

Jason

#128 lore raw

Re: [RFC PATCH v1 23/38] coco: guest: arm64: Update arm CCA guest driver

In reply to Jonathan Cameron
Quoted 8 lines · Jonathan Cameron

drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 208

Quoted 4 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1 -3
-static void __exit arm_cca_guest_exit(void)
-{
-	tsm_report_unregister(&arm_cca_tsm_ops);
+	return ret;

return devm_add_action_or_reset()

Mind you, Jason probably won't like this ;)

devm in a module __exit function? How ?

Jason

#85 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jason Gunthorpe
Quoted 35 lines · On Thu, 31 Jul 2025 09:11:33 -0300 Jason Gunthorpe <[email protected]> wrote:
Quoted 23 lines · Jonathan Cameron
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Is faux_device a better choice? I admit to not knowing entirely what it is for..

I'll go with a cautious yes to faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.

Not that many examples yet, but cpuidle-psci.c looks like a vaguely similar case to this one.

Quoted 7 lines

All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if probe() fails, which is cleaner in many ways than leaving a pointless stub behind.

Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.

Yeah, exactly. Can a TSM driver even be modular? If it has to be built in then there is no reason to do this:

Quoted 7 lines
Quoted 2 lines
Quoted 2 lines

The goal is to have tsm class device to be parented by the platform device.

IMHO the only real point of that is to trigger module autoloading.

Otherwise the tsm core should accept NULL as the parent pointer during registration, it probably already does..

If you mean create a class device with no parent, that's also something we are slowly trying to fix. Reminds me that fixing up more perf devices is still on my todo list.

Should be a child of something, so maybe that is a good reason for a faux_device here if there is nothing else to use.

Jonathan

Quoted 2 lines

Jason

#12 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Jason Gunthorpe
Quoted 5 lines · Jason Gunthorpe
Quoted 4 lines · Suzuki K Poulose
Quoted 4 lines
Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

PCI SIG supports it, kernel should support it.

ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.

Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.

Yes, this is what the VMM is supposed to do today, see [0] & [1].

Quoted 5 lines

I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for

Yes, share/private transitions do go all the way back to VMM and it is supposed to make the necessary changes to the SMMU S2 (as in [1]).

Quoted 2 lines

the RMM S2?

As for the RMM S2, the current plan is to re-use the CPU S2 managed by RMM.

Quoted 8 lines

The first option sounds fairly appealing, IMHO

Quoted 3 lines

Rather than mapping the entire memory from the host, it would be ideal if the Coco vms have some sort of a callback to "make sure the DMA wouldn't fault for a device".

Isn't that a different topic? For right now we expect that all pages are pinned and loaded into both S2s. Upon any private/shared

Actually it is. But might solve the problem for confidential VMs, where the S2 mapping is kind of pinned.

Population of S2 is a bit tricky for CVMs, as there are restrictions due to :
1) Pre-boot measurements
2) Restrictions on modifying the S2 (at least on CCA).

Thus, "the preload S2 and pin" must be done, after the "Initial images are loaded". And that becomes tricky from the Hypervisor (thinking of this, the VMM may be able to do this properly, as long as it remembers which areas where loaded).

Filling in the S2, with already populated S2 is complicated for CCA (costly, but not impossible). But the easier way is for the Realm to fault in the pages before they are used for DMA (and S2 mappings can be pinned by the hyp as default). Hence that suggestion.

Suzuki

[0] https://gitlab.arm.com/linux-arm/kvmtool-cca/-/commit/7c34972ddc
[1] https://gitlab.arm.com/linux-arm/kvmtool-cca/-/commit/ab4e654c4

Quoted 9 lines

conversion the pages should be reloaded into the appropriate S2s if required. The VM never needs to tell the hypervisor that it wants to do DMA.

There are all sorts of options here to relax this but exploring them it an entirely different project that CCA, IMHO.

Jason

#129 lore raw

Re: [RFC PATCH v1 23/38] coco: guest: arm64: Update arm CCA guest driver

In reply to Jason Gunthorpe
Quoted 11 lines · On Thu, 31 Jul 2025 09:29:48 -0300 Jason Gunthorpe <[email protected]> wrote:
Quoted 8 lines · Jonathan Cameron

drivers/virt/coco/arm-cca-guest/arm-cca.c static int arm_cca_report_new(struct tsm_report *report, void *data) · line 208

Quoted 4 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c renamed from drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +1 -3
-static void __exit arm_cca_guest_exit(void)
-{
-	tsm_report_unregister(&arm_cca_tsm_ops);
+	return ret;  

return devm_add_action_or_reset()

Mind you, Jason probably won't like this ;)

devm in a module __exit function? How ?

More coffee time...

Quoted 2 lines

Jason

#13 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Suzuki K Poulose
Quoted 8 lines · Suzuki K Poulose
Quoted 7 lines · Jason Gunthorpe
Quoted 6 lines · Suzuki K Poulose
Quoted 5 lines
Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

PCI SIG supports it, kernel should support it.

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.

Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.

Yes, this is what the VMM is supposed to do today, see [0] & [1].

Okay great!

Quoted 7 lines
Quoted 4 lines

I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for

Yes, share/private transitions do go all the way back to VMM and it is supposed to make the necessary changes to the SMMU S2 (as in [1]).

Okay, it works, but also why?

From a hypervisor perspective when using VFIO I'd like the guestmemfd to fix all the physical memory immediately, so the entire physical map is fixed and known. Backed by 1G huge pages most likely.

Is there a reason not to just dump that into the T=0 SMMU using 1G huge pages and never touch it again? The GPT provides protection?

Sure sounds appealing..

Quoted 2 lines

As for the RMM S2, the current plan is to re-use the CPU S2 managed by RMM.

Yes, but my question is if the CPU will be prepopulated

Quoted 2 lines

Actually it is. But might solve the problem for confidential VMs, where the S2 mapping is kind of pinned.

Not kind of pinned, it is pinned in the hypervisor..

Quoted 4 lines

Population of S2 is a bit tricky for CVMs, as there are restrictions due to :
1) Pre-boot measurements
2) Restrictions on modifying the S2 (at least on CCA).

I haven't dug into any of this, but I'd challenge you to try to make it run fast if the guestmemfd has a full fixed address map in 1G pages and could just dump them into the RMM efficiently once during boot.

Perhaps there are ways to optimize the measurements for huge amounts of zero'd memory.

Quoted 4 lines

Filling in the S2, with already populated S2 is complicated for CCA (costly, but not impossible). But the easier way is for the Realm to fault in the pages before they are used for DMA (and S2 mappings can be pinned by the hyp as default). Hence that suggestion.

I guess, but it's weird, kinda slow, and the RMM can never unfault them..

How will you reconstruct the 1G huge pages in the S2 if you are only populating on faults? Can you really fault the entire 1G page? If so why can't it be prepopulated?

Jason

#86 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jonathan Cameron
Quoted 3 lines · Jonathan Cameron

If you mean create a class device with no parent, that's also something we are slowly trying to fix. Reminds me that fixing up more perf devices is still on my todo list.

IIRC if you create a class device with no parent it gets placed on the virtual bus...

Do you mean we should not do that?

Quoted 2 lines

Should be a child of something, so maybe that is a good reason for a faux_device here if there is nothing else to use.

Don't see such a big difference to have it be the child of a faux device on the faux bus than to just be directly on the virtual bus?

Jason

#151 lore raw

Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report

In reply to Arto Merilainen
Quoted 15 lines · Arto Merilainen
Quoted 13 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +13 -0
+	for (int i = 0; i < interface_report->mmio_range_count; i++, mmio_range++) {
+
+		/*FIXME!! units in 4K size*/
+		range_id = FIELD_GET(TSM_INTF_REPORT_MMIO_RANGE_ID, mmio_range->range_attributes);
+
+		/* no secure interrupts */
+		if (msix_tbl_bar != -1 && range_id == msix_tbl_bar) {
+			pr_info("Skipping misx table\n");
+			continue;
+		}
+
+		if (msix_pba_bar != -1 && range_id == msix_pba_bar) {
+			pr_info("Skipping misx pba\n");
Quoted 8 lines

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 292

Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +3 -0
+			continue;
+		}
+

MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated.

Right, there are not enough BARs in most devices to give MSI its own BAR.

Quoted 4 lines

It is enough to have sufficient isolation within the
BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.

Right, this sounds bad.

Quoted 3 lines

Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?

IMHO this is a mess. The virtualization must end up putting a shared page(s) covering the MSI space in the middle of the MMIO region.

I think this should be done by fragmenting the layout in the IPA where the private MMIO is within the protected IPA space with an unmapped hole covering the MSIX registers. The acceptance process should validate this.

The MSIX registers would then be located in the shared IPA space.

A normal driver mmaping it's BAR will then crash if it tries to access the MSIX registers. This is good, we want to catch these non-secure configurations and block them.

The MSI code will have to know to compute the shared IPA alias and use that.

Jason

#175 lore raw
In reply to Aneesh Kumar K.V (Arm)
Quoted 9 lines · Aneesh Kumar K.V (Arm)

This patch series implements support for Device Assignment in the ARM CCA architecture. The code changes are based on Alp12 specification published here [1].

The code builds on the TSM framework patches posted at [2]. We add extension to that framework so that TSM is now used in both the host and the guest.

A DA workflow can be summarized as below:

Quoted 8 lines

Host:
step 1.
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind echo vfio-pci > /sys/bus/pci/devices/${DEVICE}/driver_override echo ${DEVICE} > /sys/bus/pci/drivers_probe

step 2.
echo 1 > /sys/bus/pci/devices/$DEVICE/tsm/connect

Just for my own understanding... presumably there is no ordering constraint for ARM CCA between step1 and step2, right? I.e. The connect state is independent of the bind state.

In the v4 PCI/TSM scheme the connect command is now:

echo $tsm_dev > /sys/bus/pci/devices/$DEVICE/tsm/connect

Quoted 1 line

Now in the guest we follow the below steps

I assume a signifcant amount of kvmtool magic happens here to get the TDI into a "bind capable" state, can you share that command?

I had been assuming that everyone was prototyping with QEMU. Not a problem per se, but the memory management for shared device assignment / bounce buffering has had a quite of bit of work on the QEMU side, so just curious about the difference in approach here. Like, does kvmtool support operating the device in shared mode with bounce buffering and page conversion (shared <=> private) support? In any event, happy to see mutiple simultaneous consumers of this new kernel infrastructure.

Quoted 5 lines

step 1:
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind

step 2: Move the device to TDISP LOCK state
echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/lock

Ok, so my stance has recently picked up some nuance here. As Jason mentions here:

http://lore.kernel.org/[email protected]

"However it works, it should be done before the driver is probed and remain stable for the duration of the driver attachment. From the iommu side the correct iommu domain, on the correct IOMMU instance to handle the expected traffic should be setup as the DMA API's iommu domain."

I agree with that up until the point where the implication is userspace control of the UNLOCKED->LOCKED transition. That transition requires enabling bus-mastering (BME), configuring the device into an expected state, and *then* locking the device. That means userspace is blindly hoping that the device is in a state where it will remain quiet on the bus between BME and LOCKED, and that the previous unbind left the device in a state where it is prepared to be locked again.

The BME concern may be overblown given major PCI drivers blindly set BME without validating the device is in a quiesced state, but the "device is prepped for locking" problem seems harder.

2 potential ways to solve this, but open to other ideas:

- Userspace only picks the iommu domain context for the device not the lock state. Something like:

  private > /sys/bus/pci/devices/${DEVICE}/tsm/domain

...where the default is "shared" and from that point the device can not issue DMA until a driver attaches. Driver controls UNLOCKED->LOCKED->RUN.

- Userspace is not involved in this transition and the dma mapping API is updated to allow a driver to switch the iommu domain at runtime, but only if the device has no outstanding mappings and the transition can only happen from ->probe() context. Driver controls joining secure-world-DMA and UNLOCKED->LOCKED->RUN.

Clearly the first option is less work in the kernel, but in both options the driver is in control of when BME is set relative to being ready for the LOCKED transition.

Quoted 2 lines

step 3: Moves the device to TDISP RUN state echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/accept

This has the same concern from me about userspace being in control of BME. It feels like a departure from typical expectations. At least in the case of a driver setting BME the driver's probe routine is going to get the device in order shortly and otherwise have error handlers at the ready to effect any needed recovery.

Userspace just leaves the device enabled indefinitely and hopes.

Now, the nice thing about the scheme as proposed in this set is that userspace has all the time in the world between "lock" and "accept" to talk to a verifier.

With the driver in control there would need to be something like a usermodehelper to notify userspace that the device is in the locked state and to go ahead and run the attestation while the driver waits*.

* or driver could decide to not wait, especially useful for debug and development

Quoted 2 lines

step 4: Load the driver again.
echo ${DEVICE} > /sys/bus/pci/drivers_probe

TIL drivers_probe

Maybe want to recommend:

echo ${DEVICE} > /sys/bus/pci/drivers/${DRIVER}/bind

...to users just in case there are multiple drivers loaded for the device for the "shared" vs "private" case?

Quoted 2 lines

I'm currently working against TSM v3, as TSM v4 lacks the necessary callbacks—bind, unbind, and guest_req—required for guest interactions.

For staging purposes I wanted to put the "connect" flow to bed before moving on to the guest side.

Quoted 16 lines

The implementation also makes use of RHI interfaces that fall outside the current RHI specification [5]. Once the spec is finalized, the code will be aligned accordingly.

For now, I’ve retained validate_mmio and vdev_req exit handling within KVM. This will transition to a guest_req-based mechanism once the specification is updated.

At that point, all device assignment (DA)-specific VM exits will exit directly to the VMM, and will use the guest_req ioctl to handle exit reasons. As part of this change, the handlers realm_exit_vdev_req_handler, realm_exit_vdev_comm_handler, and realm_exit_dev_mem_map_handler will be removed.

Full patchset for the kernel and kvmtool can be found at [3] and [4]

Quoted 7 lines

[1] https://developer.arm.com/-/cdn-downloads/permalink/Architectures/Armv9/DEN0137_1.1-alp12.zip

[2] https://lore.kernel.org/all/[email protected]

[3] https://git.gitlab.arm.com/linux-arm/linux-cca.git cca/tdisp-upstream-post-v1
[4] https://git.gitlab.arm.com/linux-arm/kvmtool-cca.git cca/tdisp-upstream-post-v1
[5] https://developer.arm.com/documentation/den0148/latest/

Thanks for this and the help reviewing PCI/TSM so far! I want to get this into tsm.git#staging so we can start to make hard claims ("look at the shared tree!") of hardware vendor consensus.

maintainer
#87 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jason Gunthorpe
Quoted 9 lines · Jason Gunthorpe
Quoted 3 lines · Jonathan Cameron

If you mean create a class device with no parent, that's also something we are slowly trying to fix. Reminds me that fixing up more perf devices is still on my todo list.

IIRC if you create a class device with no parent it gets placed on the virtual bus...

Quoted 7 lines

Do you mean we should not do that?

Quoted 2 lines

Should be a child of something, so maybe that is a good reason for a faux_device here if there is nothing else to use.

Don't see such a big difference to have it be the child of a faux device on the faux bus than to just be directly on the virtual bus?

Either is fine, but just never use a platform device for a non-platform-resource-backed device please.

thanks,

greg k-h

#14 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Jason Gunthorpe
Quoted 28 lines · Jason Gunthorpe
Quoted 8 lines · Suzuki K Poulose
Quoted 7 lines · Jason Gunthorpe
Quoted 6 lines · Suzuki K Poulose
Quoted 5 lines
Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

PCI SIG supports it, kernel should support it.

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

ACK. On Arm CCA, the device can access shared IPA, with T=1 transaction as long as the mapping is active in the Stage2 managed by RMM.

Right, I expect that the T=0 SMMU S2 translation is a perfect subset of the T=1 S2 rmm translation. At most pages that are not available to T=0 should be removed when making the subset.

Yes, this is what the VMM is supposed to do today, see [0] & [1].

Okay great!

Quoted 7 lines
Quoted 4 lines

I'm not sure what the plan is here on ARM though, do you expect to pre-load the entire T=0 SMMU S2 with the shared IPA aliases and rely on the GPT for protection or will the hypervisor dynamically change the T=0 SMMU S2 after each shared/private change? Same question for

Yes, share/private transitions do go all the way back to VMM and it is supposed to make the necessary changes to the SMMU S2 (as in [1]).

Quoted 8 lines

Okay, it works, but also why?

From a hypervisor perspective when using VFIO I'd like the guestmemfd
to fix all the physical memory immediately, so the entire physical map is fixed and known. Backed by 1G huge pages most likely.

Is there a reason not to just dump that into the T=0 SMMU using 1G huge pages and never touch it again? The GPT provides protection?

That is possible, once we get guest_memfd mmap support merged upstream. GPT does provide protection. The only caveat is, does the guest_memfd support this at all ? i.e., shared->private transitions with a shared mapping in place (Though this is in SMMU only, not the Host CPU pagetables)

Quoted 18 lines

Sure sounds appealing..

Quoted 2 lines

As for the RMM S2, the current plan is to re-use the CPU S2 managed by RMM.

Yes, but my question is if the CPU will be prepopulated

Quoted 2 lines

Actually it is. But might solve the problem for confidential VMs, where the S2 mapping is kind of pinned.

Not kind of pinned, it is pinned in the hypervisor..

Quoted 4 lines

Population of S2 is a bit tricky for CVMs, as there are restrictions due to :
1) Pre-boot measurements
2) Restrictions on modifying the S2 (at least on CCA).

Quoted 6 lines

I haven't dug into any of this, but I'd challenge you to try to make it run fast if the guestmemfd has a full fixed address map in 1G pages and could just dump them into the RMM efficiently once during boot.

Perhaps there are ways to optimize the measurements for huge amounts of zero'd memory.

There is. We (VMM) can choose not to "measure" the zero'd pages.

Quoted 10 lines
Quoted 4 lines

Filling in the S2, with already populated S2 is complicated for CCA (costly, but not impossible). But the easier way is for the Realm to fault in the pages before they are used for DMA (and S2 mappings can be pinned by the hyp as default). Hence that suggestion.

I guess, but it's weird, kinda slow, and the RMM can never unfault them..

How will you reconstruct the 1G huge pages in the S2 if you are only populating on faults? Can you really fault the entire 1G page? If so why can't it be prepopulated?

It is tricky to prepopulate the 1G page, as parts of the pages may be "populated" with contents. We can recreate the 1G block mapping by "FOLD" ing the leaf level tables, all the way upto 1G, after the mappings are created. We have to do that anyway for CCA.

I think we can go ahead with VMM pre-populating the entire DRAM and keeping it pinned for DA. Rather than doing this from the vfio kernel, it could be done by the VMM as it has better knowledge of the populated contents and map the rest as "unmeasured" 0s.

Suzuki

#15 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

Quoted 8 lines · Suzuki K Poulose
Quoted 2 lines

Is there a reason not to just dump that into the T=0 SMMU using 1G huge pages and never touch it again? The GPT provides protection?

That is possible, once we get guest_memfd mmap support merged upstream. GPT does provide protection. The only caveat is, does the guest_memfd support this at all ? i.e., shared->private transitions with a shared mapping in place (Though this is in SMMU only, not the Host CPU pagetables)

I don't know, we haven't got to the guestmemfd/IOMMU integration yet, which is why I ask the questions.

I think AMD and ARM would both be interested in guestmemfd <-> iommu working this way, at least.

Quoted 4 lines

I think we can go ahead with VMM pre-populating the entire DRAM and keeping it pinned for DA. Rather than doing this from the vfio kernel, it could be done by the VMM as it has better knowledge of the populated contents and map the rest as "unmeasured" 0s.

Yes, if done it should be done by the VMM and run through guestmemfd/kvm however that is agreed to.

Jason

#176 lore raw
In reply to [email protected]
Quoted 10 lines · On Thu, Jul 31, 2025 at 07:07:17PM -0700, [email protected] wrote:
Quoted 8 lines · Aneesh Kumar K.V (Arm)

Host:
step 1.
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind echo vfio-pci > /sys/bus/pci/devices/${DEVICE}/driver_override echo ${DEVICE} > /sys/bus/pci/drivers_probe

step 2.
echo 1 > /sys/bus/pci/devices/$DEVICE/tsm/connect

Quoted 7 lines

Just for my own understanding... presumably there is no ordering constraint for ARM CCA between step1 and step2, right? I.e. The connect state is independent of the bind state.

In the v4 PCI/TSM scheme the connect command is now:

echo $tsm_dev > /sys/bus/pci/devices/$DEVICE/tsm/connect

What does this do on the host? It seems to somehow prep it for VM assignment? Seems pretty strange this is here in sysfs and not part of creating the vPCI function in the VM through VFIO and iommufd?

Frankly, I'm nervous about making any uAPI whatsoever for the hypervisor side at this point. I don't think we have enough of the solution even in draft format. I'd really like your first merged TSM series to only have uAPI for the guest side where things are hopefully closer to complete..

Quoted 9 lines
Quoted 5 lines

step 1:
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind

step 2: Move the device to TDISP LOCK state
echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/lock

Ok, so my stance has recently picked up some nuance here. As Jason mentions here:

Quoted 7 lines

http://lore.kernel.org/[email protected]

"However it works, it should be done before the driver is probed and remain stable for the duration of the driver attachment. From the iommu side the correct iommu domain, on the correct IOMMU instance to handle the expected traffic should be setup as the DMA API's iommu domain."

I think it is not just the dma api, but also the MMIO registers may move location (form shared to protected IPA space for example). Meaning any attached driver is completely wrecked.

Quoted 3 lines

I agree with that up until the point where the implication is userspace control of the UNLOCKED->LOCKED transition. That transition requires enabling bus-mastering (BME),

Why? That's sad. BME should be controlled by the VM driver not the TSM, and it should be set only when a VM driver is probed to the RUN state device?

Quoted 4 lines

and *then* locking the device. That means userspace is blindly hoping that the device is in a state where it will remain quiet on the bus between BME and LOCKED, and that the previous unbind left the device in a state where it is prepared to be locked again.

Yes, but we broadly assume this already in Linux. Drivers assume their devices are quiet when they are bound the first time, we expect on unbinding a driver quiets the device before removing.

So broadly I think you can assume that a device with no driver is quiet regardless of BME.

Quoted 10 lines

2 potential ways to solve this, but open to other ideas:

- Userspace only picks the iommu domain context for the device not the lock state. Something like:

  private > /sys/bus/pci/devices/${DEVICE}/tsm/domain

...where the default is "shared" and from that point the device can not issue DMA until a driver attaches. Driver controls UNLOCKED->LOCKED->RUN.

What? Gross, no way can we let userspace control such intimate details of the kernel. The kernel must auto set based on what T=x mode the device driver binds into.

Quoted 5 lines

- Userspace is not involved in this transition and the dma mapping API is updated to allow a driver to switch the iommu domain at runtime, but only if the device has no outstanding mappings and the transition can only happen from ->probe() context. Driver controls joining secure-world-DMA and UNLOCKED->LOCKED->RUN.

I don't see why it is so complicated. The driver is unbound before it reaches T=1 so we expect the device to be quiet (bigger problems if not). When the PCI core reaches T=1 it tells the DMA API to reconfigure things for the unbound struct device. Then we bind a driver as normal.

Driver controls nothing. All existing T=0 drivers "just work" with no source changes in T=1 mode. DMA API magically hides the bounce buffering. Surely this should be the baseline target functionality from a Linux perspective?

So we should not have "driver controls" statements at all. Userspace prepares the PCI device, driver probes onto a T=1 environment and just works.

Quoted 5 lines
Quoted 2 lines

step 3: Moves the device to TDISP RUN state echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/accept

This has the same concern from me about userspace being in control of BME. It feels like a departure from typical expectations.

It is, it is architecturally broken for BME to be controlled by the TSM. BME is controlled by the guest OS driver only.

IMHO if this is a real worry (and I don't think it is) then the right answer is for physical BME to be set on during locking, but VIRTUAL BME is left off. Virtual BME is created by the hypervisor/tsm by telling the IOMMU to block DMA.

The Guest OS should not participate in this broken design, the hypervisor can set pBME automatically when the lock request comes in, and the quality of vBME emulation is left up to the implementation, but the implementation must provide at least a NOP vBME once locked.

Quoted 3 lines

Now, the nice thing about the scheme as proposed in this set is that userspace has all the time in the world between "lock" and "accept" to talk to a verifier.

Seems right to me. There should be NO trusted kernel driver bound until the verifier accepts the attestation. Anything else allows un accepted devices to attack the kernel drivers. Few kernel drivers today distrust their HW interfaces as hostile actors and security defend against them. Therefore we should be very reluctant to bind drivers to anything..

Arguably a CC secure kernel should have an allow list of audited secure drivers that can autoprobe and all other drivers must be approved by userspace in some way, either through T=1 and attestation or some customer-aware risk assumption.

From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.

Otherwise, if you say, have a TDISP capable mlx5 device and boot up the cVM in a comporomised host the host can probably completely hack your cVM by exploiting the mlx5 drivers's total trust in the HW interface while running in T=0 mode.

You must attest it and switch to T=1 before binding any driver if you care about mitigating this risk.

Quoted 3 lines

With the driver in control there would need to be something like a usermodehelper to notify userspace that the device is in the locked state and to go ahead and run the attestation while the driver waits*.

It doesn't make sense to require modification to all existing drivers in Linux! The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.

Quoted 11 lines
Quoted 2 lines

step 4: Load the driver again.
echo ${DEVICE} > /sys/bus/pci/drivers_probe

TIL drivers_probe

Maybe want to recommend:

echo ${DEVICE} > /sys/bus/pci/drivers/${DRIVER}/bind

...to users just in case there are multiple drivers loaded for the device for the "shared" vs "private" case?

Generic userspace will have a hard time to know what the driver names are..

The driver_probe option looks good to me as the default.

I'm not sure how generic code can handle "multiple drivers".. Most devices will be able to work just fine with T=0 mode with bounce buffers so we should generally not encourage people to make completely different drivers for T=0/T=1 mode.

I think what is needed is some way for userspace to trigger the "locking configuration" you mentioned, that may need a special driver, but ONLY if the userspace is sequencing the device to T=1 mode. Not sure how to make that generic, but I think so long as userspace is explicitly controlling driver binding we can punt on that solution to the userspace project :)

The real nastyness is RAS - what do you do when the device falls out of RUN, the kernel driver should pretty much explode. But lots of people would like the kernel driver to stay alive and somehow we FLR, re-attest and "resume" the kernel driver without allowing any T=0 risks. For instance you can keep your netdev and just see a lot of lost packets while the driver thrashes.

But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.

Jason

#177 lore raw
Quoted 15 lines · Jason Gunthorpe
Quoted 14 lines · On Thu, Jul 31, 2025 at 07:07:17PM -0700, [email protected] wrote:
Quoted 8 lines · Aneesh Kumar K.V (Arm)

Host:
step 1.
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind echo vfio-pci > /sys/bus/pci/devices/${DEVICE}/driver_override echo ${DEVICE} > /sys/bus/pci/drivers_probe

step 2.
echo 1 > /sys/bus/pci/devices/$DEVICE/tsm/connect

Just for my own understanding... presumably there is no ordering constraint for ARM CCA between step1 and step2, right? I.e. The connect state is independent of the bind state.

Quoted 7 lines
Quoted 3 lines

In the v4 PCI/TSM scheme the connect command is now:

echo $tsm_dev > /sys/bus/pci/devices/$DEVICE/tsm/connect

What does this do on the host? It seems to somehow prep it for VM assignment? Seems pretty strange this is here in sysfs and not part of creating the vPCI function in the VM through VFIO and iommufd?

vPCI is out of the picture at this phase.

On the host this establishes an SPDM session and sets up link encryption (IDE) with the physical device. Leave VMs out of the picture, this capability in isolation is a useful property. It addresses the similar threat model that Intel Total Memory Encryption (TME) or AMD Secure Memory Encryption (SME) go after, i.e. interposer on a physical link capturing data in flight.

With that established then one can go futher to do the full TDISP dance.

Quoted 5 lines

Frankly, I'm nervous about making any uAPI whatsoever for the hypervisor side at this point. I don't think we have enough of the solution even in draft format. I'd really like your first merged TSM series to only have uAPI for the guest side where things are hopefully closer to complete.

Aligned. I am not comfortable merging any of this until we have that end to end reliably stable for a kernel cycle or 2. The proposal is soak all the vendor solutions together in tsm.git#staging.

Now, if the guest side graduates out of that staging before the host side, I am ok with that.

Quoted 17 lines
Quoted 9 lines
Quoted 5 lines

step 1:
echo ${DEVICE} > /sys/bus/pci/devices/${DEVICE}/driver/unbind

step 2: Move the device to TDISP LOCK state
echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/lock

Ok, so my stance has recently picked up some nuance here. As Jason mentions here:

Quoted 7 lines

http://lore.kernel.org/[email protected]

"However it works, it should be done before the driver is probed and remain stable for the duration of the driver attachment. From the iommu side the correct iommu domain, on the correct IOMMU instance to handle the expected traffic should be setup as the DMA API's iommu domain."

Quoted 3 lines

I think it is not just the dma api, but also the MMIO registers may move location (form shared to protected IPA space for example). Meaning any attached driver is completely wrecked.

True.

Quoted 7 lines
Quoted 3 lines

I agree with that up until the point where the implication is userspace control of the UNLOCKED->LOCKED transition. That transition requires enabling bus-mastering (BME),

Why? That's sad. BME should be controlled by the VM driver not the TSM, and it should be set only when a VM driver is probed to the RUN state device?

To me it is an unfortunate PCI specification wrinkle that writing to the command register drops the device from RUN to ERROR. So you can LOCK without setting BME, but then no DMA.

Quoted 19 lines
Quoted 4 lines

and *then* locking the device. That means userspace is blindly hoping that the device is in a state where it will remain quiet on the bus between BME and LOCKED, and that the previous unbind left the device in a state where it is prepared to be locked again.

Yes, but we broadly assume this already in Linux. Drivers assume their devices are quiet when they are bound the first time, we expect on unbinding a driver quiets the device before removing.

So broadly I think you can assume that a device with no driver is quiet regardless of BME.

Quoted 7 lines

2 potential ways to solve this, but open to other ideas:

- Userspace only picks the iommu domain context for the device not the lock state. Something like:

  private > /sys/bus/pci/devices/${DEVICE}/tsm/domain
Quoted 7 lines
Quoted 3 lines

...where the default is "shared" and from that point the device can not issue DMA until a driver attaches. Driver controls UNLOCKED->LOCKED->RUN.

What? Gross, no way can we let userspace control such intimate details of the kernel. The kernel must auto set based on what T=x mode the device driver binds into.

Flummoxed. Any way this gets sliced, userspace is asking for "private world attach" because it alone knows that this device is acceptable, and devices need to arrive in "shared world attach" mode.

Quoted 12 lines
Quoted 5 lines

- Userspace is not involved in this transition and the dma mapping API is updated to allow a driver to switch the iommu domain at runtime, but only if the device has no outstanding mappings and the transition can only happen from ->probe() context. Driver controls joining secure-world-DMA and UNLOCKED->LOCKED->RUN.

I don't see why it is so complicated. The driver is unbound before it reaches T=1 so we expect the device to be quiet (bigger problems if not). When the PCI core reaches T=1 it tells the DMA API to reconfigure things for the unbound struct device. Then we bind a driver as normal.

Quoted 4 lines

Driver controls nothing. All existing T=0 drivers "just work" with no source changes in T=1 mode. DMA API magically hides the bounce buffering. Surely this should be the baseline target functionality from a Linux perspective?

I started this project with "all existing T=0 drivers 'just work'" as a goal and a virtue. I have been begrudgingly pulled away from it from the slow drip of complexity it appears to push into the PCI core.

Now, I suspect the number of devices that are willing to spend gates and firmware on TDISP capabilities in the near term is small. The "just works" case is saved for either an L1 VMM to hide all this from an L2 guest, or a simplified TDISP specification that actually allows an OS PCI core to handle these details in a standard way.

Quoted 3 lines

So we should not have "driver controls" statements at all. Userspace prepares the PCI device, driver probes onto a T=1 environment and just works.

The concern is neither userspace nor the PCI core have everything it needs to get the device to T=1. PCI core knows that the device is T=1 capable, but does not know how to preconfigure the device-specific lock state, needs to wait for attestation. Userpace knows how to attest/verify the device but really has no business running the device outside of binding a driver, and can not rely on the PCI core to have prepped the device's device-specific lock state.

Userspace might be able to bind a new driver that leaves the device in a lockable state on unbind, but that is not "just works" that is, "introduce a new concept of skinny TDISP setup drivers that leave devices in LOCKED state on driver unbind, so that userspace can do the work to verify the device and move it to RUN before loading the main driver that expects the device arrives already running. Also, that main driver needs to be careful not to trigger typically benign actions like touch the command register to trip the device into ERROR state, or any device-specific actions that trip ERROR state but would otherwise be benign outside of TDISP."

If locking the device was just a toggle it would be possible. As far as I can see it is a "prep+toggle" where "prep" needs a driver.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

step 3: Moves the device to TDISP RUN state echo 1 > /sys/bus/pci/devices/${DEVICE}/tsm/accept

This has the same concern from me about userspace being in control of BME. It feels like a departure from typical expectations.

It is, it is architecturally broken for BME to be controlled by the TSM. BME is controlled by the guest OS driver only.

Agree. That "accept" attribute does not belong with TSM. That is where Aneesh has it in this RFC. "Accept" as an action is the combination of device entered the LOCKED state in a configuration the verifier is willing to accept and the mechanics of triggering the LOCKED->RUN transition.

Quoted 9 lines

IMHO if this is a real worry (and I don't think it is) then the right answer is for physical BME to be set on during locking, but VIRTUAL BME is left off. Virtual BME is created by the hypervisor/tsm by telling the IOMMU to block DMA.

The Guest OS should not participate in this broken design, the hypervisor can set pBME automatically when the lock request comes in, and the quality of vBME emulation is left up to the implementation, but the implementation must provide at least a NOP vBME once locked.

I can let go of the "BME without driver" worry, but that does nothing to solve the "device specific configuration required before lock" problem.

Quoted 11 lines
Quoted 3 lines

Now, the nice thing about the scheme as proposed in this set is that userspace has all the time in the world between "lock" and "accept" to talk to a verifier.

Seems right to me. There should be NO trusted kernel driver bound until the verifier accepts the attestation. Anything else allows un accepted devices to attack the kernel drivers. Few kernel drivers today distrust their HW interfaces as hostile actors and security defend against them. Therefore we should be very reluctant to bind drivers to anything.

Quoted 4 lines

Arguably a CC secure kernel should have an allow list of audited secure drivers that can autoprobe and all other drivers must be approved by userspace in some way, either through T=1 and attestation or some customer-aware risk assumption.

Yes, today, where nothing is T=1 capable for an L1 guest*, the onus is 100% on the distribution, not the kernel. I.e. trim kernel config and set modprobe policy to prevent unwanted drivers.

* For L2 there are proposals like this, where if you already trust your paravisor also pre-trust all the devices it tells you to trust.
[1]: http://lore.kernel.org/[email protected]

Quoted 4 lines

From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.

Agree, for PCI it would be simple to set a no-auto-probe policy for T=1 capable devices.

Quoted 7 lines

Otherwise, if you say, have a TDISP capable mlx5 device and boot up the cVM in a comporomised host the host can probably completely hack your cVM by exploiting the mlx5 drivers's total trust in the HW interface while running in T=0 mode.

You must attest it and switch to T=1 before binding any driver if you care about mitigating this risk.

Yes, userspace must have a chance to say "no" before a driver attempts to launch DMA to private memory after secrets have been deployed to the TVM.

Quoted 6 lines
Quoted 3 lines

With the driver in control there would need to be something like a usermodehelper to notify userspace that the device is in the locked state and to go ahead and run the attestation while the driver waits*.

It doesn't make sense to require modification to all existing drivers in Linux!

I do not want to burden the PCI core with TDISP compatibility hacks and workarounds if it turns out only a small handful of devices ever deploy a first generation TDISP Device Security Manager (DSM). L1 aiding L2, or TDISP simplicity improvements to allow the PCI core to handle this in a non-broken way, are what I expect if secure device assignment takes off.

Quoted 3 lines

The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.

Do you agree that "device-specific-prep+lock" is the problem to solve?

Quoted 29 lines
Quoted 11 lines
Quoted 2 lines

step 4: Load the driver again.
echo ${DEVICE} > /sys/bus/pci/drivers_probe

TIL drivers_probe

Maybe want to recommend:

echo ${DEVICE} > /sys/bus/pci/drivers/${DRIVER}/bind

...to users just in case there are multiple drivers loaded for the device for the "shared" vs "private" case?

Generic userspace will have a hard time to know what the driver names are..

The driver_probe option looks good to me as the default.

I'm not sure how generic code can handle "multiple drivers".. Most devices will be able to work just fine with T=0 mode with bounce buffers so we should generally not encourage people to make completely different drivers for T=0/T=1 mode.

I think what is needed is some way for userspace to trigger the "locking configuration" you mentioned, that may need a special driver, but ONLY if the userspace is sequencing the device to T=1 mode. Not sure how to make that generic, but I think so long as userspace is explicitly controlling driver binding we can punt on that solution to the userspace project :)

Quoted 6 lines

The real nastyness is RAS - what do you do when the device falls out of RUN, the kernel driver should pretty much explode. But lots of people would like the kernel driver to stay alive and somehow we FLR, re-attest and "resume" the kernel driver without allowing any T=0 risks. For instance you can keep your netdev and just see a lot of lost packets while the driver thrashes.

Ideally the RUN->ERROR->UNLOCKED->LOCKED->RUN recovery can fit into the existing 'struct pci_error_handlers' regime in some farther out future.

It was a "fun" discovery to see that virtual AER injection does not exist in QEMU (at least last time I checked) and assigned devices that throw physical AER events just kill the VM.

Quoted 4 lines

But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.

Hmm, having trouble not reading that back supporting my argument above:

Realistically few devices support TDISP lets require enhanced drivers to opt-into TDISP for the time being.

#88 lore raw

Re: [RFC PATCH v1 11/38] KVM: arm64: CCA: register host tsm platform device

In reply to Jason Gunthorpe
Quoted 29 lines · Jason Gunthorpe
Quoted 28 lines · Jonathan Cameron
Quoted 11 lines · On Wed, 30 Jul 2025 14:12:26 +0530 "Aneesh Kumar K.V"…
Quoted 9 lines · Jason Gunthorpe
Quoted 4 lines · Jonathan Cameron
Quoted 1 line

+static struct platform_device cca_host_dev = {

Hmm. Greg is getting increasingly (and correctly in my view) grumpy with platform devices being registered with no underlying resources etc as glue layers. Maybe some of that will come later.

Is faux_device a better choice? I admit to not knowing entirely what it is for..

I'll go with a cautious yes to faux_device. This case of a glue device with no resources and no reason to be on a particular bus was definitely the intent but I'm not 100% sure without trying it that we don't run into any problems.

Not that many examples yet, but cpuidle-psci.c looks like a vaguely similar case to this one.

All it really does is move the location of the device and smash together the device registration with probe/remove. That means the device disappears if probe() fails, which is cleaner in many ways than leaving a pointless stub behind.

Quoted 5 lines
Quoted 2 lines

Maybe it isn't appropriate it if is actually useful to rmmod/modprobe the driver.

Yeah, exactly. Can a TSM driver even be modular? If it has to be built in then there is no reason to do this:

For example, CRYPTO_DEV_CCP_DD, the AMD PCI device driver that will call tsm_register(), is already modular.

Quoted 4 lines
Quoted 2 lines
Quoted 2 lines

The goal is to have tsm class device to be parented by the platform device.

IMHO the only real point of that is to trigger module autoloading.

Right. For TDX, and I expect CCA as well, the arch code that knows that PCI/TSM functionality is available and can register a device, may be running too early to attach a driver to that device.

I.e. I would like to just use faux_device, but without the ability to do EPROBE_DEFER, for example to await the plaform IOMMU driver. It needs to move to its own bus so the attach event can be handled at a better time.

Quoted 2 lines

Otherwise the tsm core should accept NULL as the parent pointer during registration, it probably already does..

Yes, NULL @parent "just works" with tsm_register().

However, I expect all tsm_register() callers to be from modular drivers.

author maintainer
#16 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Jason Gunthorpe
Quoted 45 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe

arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56

Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5 -0
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
 	return crypt_ops->decrypt(addr, numpages);
 }
 EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;

Is this OK? I see code like this:

static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)

{
	if (force_dma_unencrypted(dev))
		return phys_to_dma_unencrypted(dev, phys);
	return phys_to_dma(dev, phys);

What are the ARM rules for generating dma addreses?

1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"

2) Device is T=1, memory is encrypted, use the phys_to_dma() normally

3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?

Quoted 5 lines
Quoted 2 lines

Can you document this in a comment?

If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.

No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.

Quoted 6 lines
Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

PCI SIG supports it, kernel should support it.

Would we also need a separate DMA allocation API for allocating addresses intended to be shared with the non-secure hypervisor?

Are there any existing drivers in the kernel that already require such shared allocations, which I could use as a reference?

-aneesh

author maintainer
#45 lore raw

Re: [RFC PATCH v1 08/38] iommufd/tsm: Add tsm_op iommufd ioctls

In reply to Jonathan Cameron
Quoted 139 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:45 +0530 "Aneesh Kumar K.V (Arm)"…

Add operations bind and unbind used to bind a TDI to the secure guest.

Hi Aneesh,

I'm mostly reading this to get head around it rather than fully review at this point.

A few things inline though that I noticed whilst doing so.

Jonathan

Quoted 26 lines
---
 drivers/iommu/iommufd/iommufd_private.h |  1 +
 drivers/iommu/iommufd/main.c            |  3 ++
 drivers/iommu/iommufd/viommu.c          | 50 +++++++++++++++++++++++++
 drivers/vfio/pci/vfio_pci_core.c        | 10 +++++
 include/uapi/linux/iommufd.h            | 18 +++++++++
 5 files changed, 82 insertions(+)
drivers/iommu/iommufd/iommufd_private.h +1 -0
index fce68714c80f..e08186f1d102 100644
@@ -697,6 +697,7 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj);
 void iommufd_vdevice_abort(struct iommufd_object *obj);
 int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd);
 void iommufd_hw_queue_destroy(struct iommufd_object *obj);
+int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd);
 
 static inline struct iommufd_vdevice *
 iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id)
drivers/iommu/iommufd/viommu.c +0 -0
index 59f1e1176f7f..c934312e5397 100644

drivers/iommu/iommufd/viommu.c void iommufd_vdevice_abort(struct iommufd_object *obj) · line 165

Quoted 4 lines
drivers/iommu/iommufd/viommu.c +1 -0
@@ -162,6 +162,9 @@ void iommufd_vdevice_abort(struct iommufd_object *obj)
 
 	lockdep_assert_held(&idev->igroup->lock);
 
+#ifdef CONFIG_TSM

Can we use stubs for some of this stuff so we don't need ifdefs in as many places.

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 479

Quoted 12 lines
drivers/iommu/iommufd/viommu.c +5 -0
+	tsm_unbind(idev->dev);
+#endif
 	if (vdev->destroy)
 		vdev->destroy(vdev);
 	/* xa_cmpxchg is okay to fail if alloc failed xa_cmpxchg previously */
@@ -471,3 +474,50 @@ int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd)
 	iommufd_put_object(ucmd->ictx, &viommu->obj);
 	return rc;
 }
+
+#ifdef CONFIG_TSM
+int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd)

Might want to split this out to a separate c file and use stubs in a header to keep the code clean here.

Quoted 9 lines
drivers/iommu/iommufd/viommu.c +9 -0
+{
+	struct iommu_vdevice_tsm_op *cmd = ucmd->cmd;
+	struct iommufd_vdevice *vdev;
+	struct kvm *kvm;
+	int rc = -ENODEV;
+
+	if (cmd->flags)
+		return -EOPNOTSUPP;
+

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 498

Quoted 10 lines
drivers/iommu/iommufd/viommu.c +10 -0
+	vdev = container_of(iommufd_get_object(ucmd->ictx, cmd->vdevice_id,
+					       IOMMUFD_OBJ_VDEVICE),
+			    struct iommufd_vdevice, obj);
+	if (IS_ERR(vdev))
+		return PTR_ERR(vdev);
+
+	kvm = vdev->viommu->kvm_filp->private_data;
+	if (kvm) {
+		/*
+		 * tsm layer will make take care of parallel calls to tsm_bind/unbind

Wrap comment to say under 80 chars. Or if file goes higher, use a single line comment.

tsm layer will take care ...

(stray 'make')

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 506

Quoted 8 lines
drivers/iommu/iommufd/viommu.c +8 -0
+		 */
+		if (cmd->op == IOMMU_VDEICE_TSM_BIND)
+			rc = tsm_bind(vdev->idev->dev, kvm, vdev->virt_id);
+		else if (cmd->op == IOMMU_VDEICE_TSM_UNBIND)
+			rc = tsm_unbind(vdev->idev->dev);
+
+		if (rc) {
+			rc = -ENODEV;

If we want to eat an error code coming from elsewhere, maybe a comment on why?

drivers/iommu/iommufd/viommu.c int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd) · line 510

Quoted 4 lines
drivers/iommu/iommufd/viommu.c +4 -0
+			goto out_put_vdev;
+		}
+	} else {
+		goto out_put_vdev;

If this always skips the next line, does that imply that line should have been under if (kvm)? Maybe this makes more sense in later patches - if so ignore this comment.

Quoted 18 lines
drivers/iommu/iommufd/viommu.c +13 -0
+	}
+	rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
+
+out_put_vdev:
+	iommufd_put_object(ucmd->ictx, &vdev->obj);
+	return rc;
+}
+#else /* !CONFIG_TSM */
+int iommufd_vdevice_tsm_op_ioctl(struct iommufd_ucmd *ucmd)
+{
+	return -ENODEV;
+}
+#endif
drivers/vfio/pci/vfio_pci_core.c +0 -0
index bee3cf3226e9..afdb39c6aefd 100644

drivers/vfio/pci/vfio_pci_core.c void vfio_pci_core_close_device(struct vfio_device *core_vdev) · line 698

Quoted 5 lines
drivers/vfio/pci/vfio_pci_core.c +2 -0
@@ -694,6 +694,16 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev)
 #if IS_ENABLED(CONFIG_EEH)
 	eeh_dev_release(vdev->pdev);
 #endif
+
+#if 0

If you really need to do this, add a comment on why if 0

Quoted 10 lines

drivers/vfio/pci/vfio_pci_core.c void vfio_pci_core_close_device(struct vfio_device *core_vdev) · line 708

Quoted 10 lines
drivers/vfio/pci/vfio_pci_core.c +8 -0
+	/*
+	 * destroy vdevice which involves tsm unbind before we disable pci disable
+	 * A MSE/BME clear will transition the device to error state.
+	 */
+	if (core_vdev->iommufd_device)
+		iommufd_device_tombstone_vdevice(core_vdev->iommufd_device);
+#endif
+
 	vfio_pci_core_disable(vdev);
 

This is something I’d like to get feedback on. According to the TSM specification, we’re required to unlock before clearing MSE/BME via calling vfio_pci_core_disable(vdev).

However, in the current iommufd branch, we seem to call vdevice_destroy a bit too late in the sequence to meet this requirement.

Quoted 27 lines
Quoted 9 lines
drivers/vfio/pci/vfio_pci_core.c +0 -0
 	mutex_lock(&vdev->igate);
include/uapi/linux/iommufd.h +0 -0
index 9014c61a97d4..8b1fbf1ef25c 100644
@@ -57,6 +57,7 @@ enum {
 
 /**

include/uapi/linux/iommufd.h enum iommu_veventq_flag { · line 1133

Quoted 6 lines
include/uapi/linux/iommufd.h +3 -0
@@ -1127,6 +1128,23 @@ enum iommu_veventq_flag {
 	IOMMU_VEVENTQ_FLAG_LOST_EVENTS = (1U << 0),
 };
 
+/**
+ * struct iommu_vdevice_tsm_OP - ioctl(IOMMU_VDEVICE_TSM_OP)
+ * @size: sizeof(struct iommu_vdevice_tsm_OP)

_op I guess?

Quoted 9 lines
include/uapi/linux/iommufd.h +9 -0
+ * @op: Either TSM_BIND or TSM_UNBIMD
+ * @flags: Must be 0
+ * @vdevice_id: Object handle for the vDevice. Returned from IOMMU_VDEVICE_ALLOC
+ */
+struct iommu_vdevice_tsm_op {
+	__u32 size;
+	__u32 op;
+	__u32 flags;
+	__u32 vdevice_id;
Quoted 8 lines

include/uapi/linux/iommufd.h enum iommu_veventq_flag { · line 1150

Quoted 8 lines
include/uapi/linux/iommufd.h +5 -0
+};
+#define IOMMU_VDEVICE_TSM_OP	_IO(IOMMUFD_TYPE, IOMMUFD_CMD_VDEVICE_TSM_OP)
+#define IOMMU_VDEICE_TSM_BIND		0x1
+#define IOMMU_VDEICE_TSM_UNBIND		0x2
+
 /**
  * struct iommufd_vevent_header - Virtual Event Header for a vEVENTQ Status
  * @flags: Combination of enum iommu_veventq_flag

Thanks for the review comments. I'll update the patch with the suggested changes.

-aneesh

author maintainer
#99 lore raw

Re: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm

In reply to Jonathan Cameron
Quoted 144 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:50 +0530 "Aneesh Kumar K.V (Arm)"…

Create the realm physical device with RMM.

Hi Aneesh,

Various small things inline.

Jonathan

Quoted 30 lines
---
 arch/arm64/include/asm/rmi_cmds.h        |  31 +++++
 arch/arm64/include/asm/rmi_smc.h         |  72 ++++++++++-
 drivers/virt/coco/arm-cca-host/Makefile  |   2 +-
 drivers/virt/coco/arm-cca-host/arm-cca.c |  10 +-
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 150 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |   5 +
 6 files changed, 267 insertions(+), 3 deletions(-)
 create mode 100644 drivers/virt/coco/arm-cca-host/rmm-da.c
arch/arm64/include/asm/rmi_cmds.h +12 -0
index ef53147c1984..f0817bd3bab4 100644
@@ -505,4 +505,35 @@ static inline int rmi_rtt_unmap_unprotected(unsigned long rd,
 	return res.a0;
 }
 
+static inline unsigned long rmi_pdev_aux_count(unsigned long flags, u64 *aux_count)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_AUX_COUNT, flags, &res);
+
+	*aux_count = res.a1;
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_create(unsigned long pdev_phys,
+					    unsigned long pdev_params_phys)

arch/arm64/include/asm/rmi_cmds.h static inline int rmi_rtt_unmap_unprotected(unsigned long rd, · line 529

Quoted 10 lines
arch/arm64/include/asm/rmi_cmds.h +10 -0
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_CREATE,
+			     pdev_phys, pdev_params_phys, &res);
+
+	return res.a0;
+}
+
+static inline unsigned long rmi_pdev_get_state(unsigned long pdev_phys, unsigned long *state)

I'd use the enum for the state type and range check in here.

Quoted 15 lines
arch/arm64/include/asm/rmi_cmds.h +9 -0
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(SMC_RMI_PDEV_GET_STATE, pdev_phys, &res);
+
+	*state = res.a1;
+	return res.a0;
+}
+
 #endif /* __ASM_RMI_CMDS_H */
arch/arm64/include/asm/rmi_smc.h +0 -0
index 42708d500048..a84ed61e5001 100644

arch/arm64/include/asm/rmi_smc.h line 29

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +0 -1
@@ -26,7 +26,7 @@
 #define SMC_RMI_DATA_CREATE		SMC_RMI_CALL(0x0153)
 #define SMC_RMI_DATA_CREATE_UNKNOWN	SMC_RMI_CALL(0x0154)
 #define SMC_RMI_DATA_DESTROY		SMC_RMI_CALL(0x0155)
-

Probably keep

arch/arm64/include/asm/rmi_smc.h line 29

Quoted 1 line
arch/arm64/include/asm/rmi_smc.h +1 -0
+#define SMC_RMI_PDEV_AUX_COUNT		SMC_RMI_CALL(0x0156)

and add one here as different groups.

Quoted 14 lines
arch/arm64/include/asm/rmi_smc.h +3 -0
 #define SMC_RMI_REALM_ACTIVATE		SMC_RMI_CALL(0x0157)
 #define SMC_RMI_REALM_CREATE		SMC_RMI_CALL(0x0158)
 #define SMC_RMI_REALM_DESTROY		SMC_RMI_CALL(0x0159)
@@ -47,6 +47,9 @@
 #define SMC_RMI_RTT_INIT_RIPAS		SMC_RMI_CALL(0x0168)
 #define SMC_RMI_RTT_SET_RIPAS		SMC_RMI_CALL(0x0169)
 
+#define SMC_RMI_PDEV_CREATE             SMC_RMI_CALL(0x0176)
+#define SMC_RMI_PDEV_GET_STATE		SMC_RMI_CALL(0x0178)
+
 #define RMI_ABI_MAJOR_VERSION	1
 #define RMI_ABI_MINOR_VERSION	0
 

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 274

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +1 -0
@@ -268,4 +271,71 @@ struct rec_run {
 	struct rec_exit exit;
 };
 
+enum rmi_pdev_state {

Type never used, but I think it should be. See above.

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 278

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4 -0
+	RMI_PDEV_NEW,
+	RMI_PDEV_NEEDS_KEY,
+	RMI_PDEV_HAS_KEY,
+	RMI_PDEV_READY,

Seems someone put another state in here, but I guess you'll update when the rest of the stack catches up.
RMI_PDEV_IDE_RESETTING,

Maybe throw a comment here for now.

Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +4 -0
+	RMI_PDEV_COMMUNICATING,
+	RMI_PDEV_STOPPING,
+	RMI_PDEV_STOPPED,
+	RMI_PDEV_ERROR,

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 292

Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10 -0
+};
+
+#define MAX_PDEV_AUX_GRANULES	32
+#define MAX_IOCOH_ADDR_RANGE	16
+#define MAX_FCOH_ADDR_RANGE	4
+
+#define RMI_PDEV_SPDM_TRUE	BIT(0)
+#define RMI_PDEV_IDE_TRUE	BIT(1)
+#define RMI_PDEV_FOCOH		BIT(2)
+#define RMI_PDEV_P2P_STREAM	BIT(3)

I'd stick flags in the name (assuming this is
RmiPDevFlags? I'm not sure as the spec I could
find has different usages other after BIT(!))

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 299

Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +7 -0
+
+#define RMI_HASH_SHA_256	0
+#define RMI_HASH_SHA_512	1
+
+struct rmi_pdev_addr_range {
+	unsigned long base;
+	unsigned long top;

Whilst we only care about platforms where this is u64, maybe just make that explicit so we can trivially see this matches the spec?

Quoted 8 lines

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 302

Quoted 3 lines
arch/arm64/include/asm/rmi_smc.h +3 -0
+};
+
+struct rmi_pdev_params {

Reference? Looks like B4.4.25 RmiPdevParams type in alp15. Though as there are some fields missing I guess this chagned.

This patch series is currently based on the ALP12 specification, available here:
https://developer.arm.com/-/cdn-downloads/permalink/Architectures/Armv9/DEN0137_1.1-alp12.zip

The next revision of the series will be updated to align with the ALP15 specification.

Quoted 35 lines
Quoted 4 lines
arch/arm64/include/asm/rmi_smc.h +3 -0
+	union {
+		struct {
+			u64 flags;

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 315

Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10 -0
+			u64 pdev_id;
+			u64 segment_id;
+			u64 ecam_addr;
+			u64 root_id;
+			u64 cert_id;
+			u64 rid_base;
+			u64 rid_top;
+			u64 hash_algo;
+			u64 num_aux;
+			u64 ide_sid;

Called ncoh_ide_side in the alpha 15. Maybe match that unless it is changing name in future version.

Quoted 17 lines
arch/arm64/include/asm/rmi_smc.h +17 -0
+			u64 ncoh_num_addr_range;
+			u64 coh_num_addr_range;
+		};
+		u8 padding1[0x100];
+	};
+
+	union { /* 0x100 */
+		u64 aux_granule[MAX_PDEV_AUX_GRANULES];
+		u8 padding2[0x100];
+	};
+
+	union { /* 0x200 */
+		struct {
+			struct rmi_pdev_addr_range ncoh_addr_range[MAX_IOCOH_ADDR_RANGE];
+		};
+		u8 padding3[0x100];
+	};
Quoted 9 lines

arch/arm64/include/asm/rmi_smc.h struct rec_run { · line 338

Quoted 6 lines
arch/arm64/include/asm/rmi_smc.h +6 -0
+	union { /* 0x300 */
+		struct {
+			struct rmi_pdev_addr_range coh_addr_range[MAX_FCOH_ADDR_RANGE];
+		};
+		u8 padding4[0x100];
+	};

Maybe pad out the rest? Mostly so we can see here that it is 4k.

ok

Quoted 167 lines
Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +2 -0
+};
+
 #endif /* __ASM_RMI_SMC_H */
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
index c8b0e6db1f47..84d97dd41191 100644

drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 127

Quoted 4 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -1
@@ -124,7 +124,15 @@ static int cca_tsm_connect(struct pci_dev *pdev)
 	rc = tsm_ide_stream_register(pdev, ide);
 	if (rc)
 		goto err_tsm;
-

Tidy up. I'd leave it.

Quoted 18 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +9 -0
+	/*
+	 * Take a module reference so that we won't call unregister
+	 * without rme_unasign_device
+	 */
+	if (!try_module_get(THIS_MODULE)) {
+		rc = -ENXIO;
+		goto err_tsm;
+	}
+	rme_asign_device(pdev);
 	/*
 	 * Once ide is setup enable the stream at endpoint
 	 * Root port will be done by RMM
drivers/virt/coco/arm-cca-host/rmm-da.c new file +0 -0
new file mode 100644
index 000000000000..426e530ac182

drivers/virt/coco/arm-cca-host/rmm-da.c line 41

Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4 -0
@@ -0,0 +1,150 @@
 
 
+
+static int init_pdev_params(struct pci_dev *pdev, struct rmi_pdev_params *params)
+{
+	void *aux;

Probably makes sense to reduce scope of aux to within the loop.

drivers/virt/coco/arm-cca-host/rmm-da.c line 45

Quoted 4 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4 -0
+	int rid, ret, i;
+	phys_addr_t aux_phys;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct pci_dev *rp = pcie_find_root_port(pdev);

Only used in one place for now anyway. Probably better to move it down there.

drivers/virt/coco/arm-cca-host/rmm-da.c line 48

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	struct pci_config_window *cfg = pdev->bus->sysdata;
+
+	dsc_pf0 = to_cca_dsc_pf0(pdev);

I'd do that at declaration. Seems little advantage in doing it down here.

drivers/virt/coco/arm-cca-host/rmm-da.c line 58

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+	/* assign the ep device with RMM */
+	rid = pci_dev_id(pdev);
+	params->pdev_id = rid;
+	/* slot number for certificate chain */
+	params->cert_id = 0;
+	/* io coherent spdm/ide and non p2p */
+	params->flags = RMI_PDEV_SPDM_TRUE | RMI_PDEV_IDE_TRUE;
+	params->ide_sid = dsc_pf0->sel_stream->stream_id;
+	params->hash_algo = RMI_HASH_SHA_256;
+	/* use the rid and MMIO resources of the epdev */

Spell out epdev or associate it with the pdev here more clearly.

drivers/virt/coco/arm-cca-host/rmm-da.c line 61

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	params->rid_top = params->rid_base = rid;
+	params->ecam_addr = cfg->res.start;
+	params->root_id = pci_dev_id(rp);
	params->root_id = pci_dev_id(pcie_find_root_port(pdev));

to me acts as better documentation fo what is going on than using the local variable rp.

drivers/virt/coco/arm-cca-host/rmm-da.c line 71

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+
+	params->ncoh_num_addr_range = pci_res_to_pdev_addr(params->ncoh_addr_range,
+							    ARRAY_SIZE(params->ncoh_addr_range),
+							    pdev->resource,
+							    DEVICE_COUNT_RESOURCE);
+
+	rmi_pdev_aux_count(params->flags, &params->num_aux);
+	pr_debug("%s using %ld pdev aux granules\n", __func__, (unsigned long)params->num_aux);
+	dsc_pf0->num_aux = params->num_aux;
+	for (i = 0; i < params->num_aux; i++) {
		void *aux = (void *)__get_free_page(GFP_KERNEL);
Quoted 9 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +9 -0
+		aux = (void *)__get_free_page(GFP_KERNEL);
+		if (!aux) {
+			ret = -ENOMEM;
+			goto err_free_aux;
+		}
+
+		aux_phys = virt_to_phys(aux);
+		if (rmi_granule_delegate(aux_phys)) {
+			ret = -ENXIO;

drivers/virt/coco/arm-cca-host/rmm-da.c line 90

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+			free_page((unsigned long)aux);
+			goto err_free_aux;
+		}
+		params->aux_granule[i] = aux_phys;
+		dsc_pf0->aux[i] = aux;
+	}
+	return 0;
+
+err_free_aux:
+	free_aux_pages(i, dsc_pf0->aux[i]);

I think you want

	free_aux_pages(i, dsc_pf0->aux);

Assuming this is supposed to unwind the loop above.

drivers/virt/coco/arm-cca-host/rmm-da.c line 93

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	return -ENOMEM;
+}
+

Trivial: Single line probably fine here.

drivers/virt/coco/arm-cca-host/rmm-da.c line 104

Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11 -0
+
+int rme_asign_device(struct pci_dev *pci_dev)
+{
+	int ret;
+	void *rmm_pdev;
+	unsigned long state;
+	phys_addr_t rmm_pdev_phys;
+	struct rmi_pdev_params *params;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+
+	dsc_pf0 = to_cca_dsc_pf0(pci_dev);

Might as well save a line with

	struct cca_host_dsc_pf0 *dsc_pf0 = to_cca_dsc_pf0(pci_dev);

drivers/virt/coco/arm-cca-host/rmm-da.c line 107

Quoted 3 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +3 -0
+	rmm_pdev = (void *)get_zeroed_page(GFP_KERNEL);
+	if (!rmm_pdev) {
+		ret = -ENOMEM;

return -ENOMEM;

Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +11 -0
+		goto err_out;
+	}
+
+	rmm_pdev_phys = virt_to_phys(rmm_pdev);
+	if (rmi_granule_delegate(rmm_pdev_phys)) {
+		ret = -ENXIO;
+		goto err_free_pdev;
+	}
+
+	params = (struct rmi_pdev_params *)get_zeroed_page(GFP_KERNEL);
+	if (!params) {

drivers/virt/coco/arm-cca-host/rmm-da.c line 128

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+		ret = -ENOMEM;
+		goto err_granule_undelegate;
+	}
+
+	ret = init_pdev_params(pci_dev, params);
+	if (ret)
+		goto err_free_params;
+
+	ret = rmi_pdev_create(rmm_pdev_phys, virt_to_phys(params));
+	pr_info("rmi_pdev_create(0x%llx, 0x%llx): %d\n", rmm_pdev_phys, virt_to_phys(params), ret);

RFC, but even so I'd demote to debug and use dynamic debug stuff to enable them for your testing.

Quoted 12 lines

drivers/virt/coco/arm-cca-host/rmm-da.c line 134

Quoted 6 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +6 -0
+	if (ret)
+		goto err_free_aux;
+
+	rmi_pdev_get_state(rmm_pdev_phys, &state);
+	if (state != RMI_PDEV_NEW)
+		goto err_free_aux;

Nothing to unwind in rmi_pdev_create()? We've told the RMM about it then we blow away it's resources. Seems unwise! Maybe this is cleaned up elsewhere but if so a comment is probably good.

pdev_create is expected to set the device state to RMI_PDEV_NEW, so the subsequent check is redundant if rmi_pdev_create returns success. I will drop that.

Quoted 36 lines
Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +4 -0
+
+	dsc_pf0->rmm_pdev = rmm_pdev;
+	free_page((unsigned long)params);
+	return 0;

drivers/virt/coco/arm-cca-host/rmm-da.c line 148

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+
+err_free_aux:
+	free_aux_pages(dsc_pf0->num_aux, dsc_pf0->aux);
+err_free_params:
+	free_page((unsigned long)params);
+err_granule_undelegate:
+	rmi_granule_undelegate(rmm_pdev_phys);
+err_free_pdev:
+	free_page((unsigned long)rmm_pdev);
+err_out:

One of my favourite nitpicks. Why not just return if nothing to do? That tends to save a reviewer a bit of scrolling when checking error paths do correct unwinding.

Quoted 16 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2 -0
+	return ret;
+}
drivers/virt/coco/arm-cca-host/rmm-da.h +4 -0
index 840cb584acdd..179ba68f2430 100644
@@ -14,6 +14,10 @@
 struct cca_host_dsc_pf0 {
 	struct pci_tsm_pf0 pci;
 	struct pci_ide *sel_stream;
+
+	void *rmm_pdev;
+	int num_aux;
+	void *aux[MAX_PDEV_AUX_GRANULES];
 };
 
Quoted 7 lines

drivers/virt/coco/arm-cca-host/rmm-da.h static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev) · line 34

Quoted 7 lines
drivers/virt/coco/arm-cca-host/rmm-da.h +1 -0
 static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
@@ -26,4 +30,5 @@ static inline struct cca_host_dsc_pf0 *to_cca_dsc_pf0(struct pci_dev *pdev)
 	return container_of(tsm, struct cca_host_dsc_pf0, pci.tsm);
 }
 
+int rme_asign_device(struct pci_dev *pdev);
 #endif

Thanks for the review comments. I'll update the patch with the suggested changes.

-aneesh

author maintainer
#101 lore raw

Re: [RFC PATCH v1 13/38] coco: host: arm64: Create a PDEV with rmm

In reply to Arto Merilainen
Quoted 18 lines · Arto Merilainen

drivers/virt/coco/arm-cca-host/rmm-da.c line 21

Quoted 10 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-host/rmm-da.c new file +10 -0
+static int pci_res_to_pdev_addr(struct rmi_pdev_addr_range *pdev_addr,
+				unsigned int naddr, struct resource *res,
+				unsigned int nres)
+{
+	int i, j;
+
+	for (i = 0, j = 0; i < naddr && j < nres; j++) {
+		if (res[j].flags & IORESOURCE_MEM) {
+			pdev_addr[i].base = res[j].start;
+			pdev_addr[i].top  = res[j].end;

I think there is an off-by-one bug in res[j].end. As per the RMM specification the base address is inclusive and the top address is exclusive. Both res[j].start and res[j].end are inclusive, and hence res[j].end seems wrong.

Quoted 6 lines

drivers/virt/coco/arm-cca-host/rmm-da.c line 59

Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c new file +2 -0
+	/* use the rid and MMIO resources of the epdev */
+	params->rid_top = params->rid_base = rid;

Similar issue here. As per the specification the rid_base is inclusive and the rid_top exclusive.

Thanks for the review comments. I'll update the patch with the suggested changes.

-aneesh

#17 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Aneesh Kumar K.V
Quoted 51 lines · Aneesh Kumar K.V
Quoted 49 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe

arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56

Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5 -0
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
 	return crypt_ops->decrypt(addr, numpages);
 }
 EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;

Is this OK? I see code like this:

static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)

{
	if (force_dma_unencrypted(dev))
		return phys_to_dma_unencrypted(dev, phys);
	return phys_to_dma(dev, phys);

What are the ARM rules for generating dma addreses?

1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"

2) Device is T=1, memory is encrypted, use the phys_to_dma() normally

3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?

Quoted 5 lines
Quoted 2 lines

Can you document this in a comment?

If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.

No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.

Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

Quoted 8 lines
Quoted 2 lines

PCI SIG supports it, kernel should support it.

Would we also need a separate DMA allocation API for allocating addresses intended to be shared with the non-secure hypervisor?

Are there any existing drivers in the kernel that already require such shared allocations, which I could use as a reference?

The most likely case in the near term is PCI P2P to shared MMIO.

I don't know any way to allocate shared memory in a driver??

At the bare minimum this patch should be documenting the correct architecture and outlining any gaps in the current implementation.

I also don't really understand what the above code is even doing.. Isn't the design on ARM that the IPA always encodes the shared/private in the top bit?

How do we get a shared page that does not already have a phys_addr_t in the shared IPA? Shouldn't the kernel have switched to the shared IPA alias when it returned the swiotlb buffer? eg why do we need to do:

#define dma_addr_unencrypted(x) ((x) | PROT_NS_SHARED)

At all?

Suzuki ?

Jason

#178 lore raw
In reply to [email protected]
Quoted 6 lines · On Fri, Aug 01, 2025 at 02:19:54PM -0700, [email protected] wrote:

On the host this establishes an SPDM session and sets up link encryption (IDE) with the physical device. Leave VMs out of the picture, this capability in isolation is a useful property. It addresses the similar threat model that Intel Total Memory Encryption (TME) or AMD Secure Memory Encryption (SME) go after, i.e. interposer on a physical link capturing data in flight.

Okay, maybe connect is not an intuitive name for opening IDE sessions..

Quoted 3 lines

I started this project with "all existing T=0 drivers 'just work'" as a goal and a virtue. I have been begrudgingly pulled away from it from the slow drip of complexity it appears to push into the PCI core.

Do you have some examples? I don't really see what complexity there is if the solution it simply not auto bind any drivers to TDISP capable devices and userspace is responsible to manually bind a driver once it has reached T=1.

This seems like the minimum possible simplicitly for the kernel as simply everything is managed by userspace, and there is really no special kernel behavior beyond switching the DMA API of an unbound driver on the T=0/1 change.

Quoted 2 lines

The concern is neither userspace nor the PCI core have everything it needs to get the device to T=1.

Disagree, I think userspace can have everything. It may need some per-device userspace support in difficult cases, but userspace can deal with it..

Quoted 2 lines

PCI core knows that the device is T=1 capable, but does not know how to preconfigure the device-specific lock state,

Userspace can do this. Can we define exactly what is needed to do this "pre-configure the device specific lock state"? At the very worst, for the most poorly designed device, userspace would have to bind a T=0 driver and then unbind it.

Again, I am trying to make something simple for the kernel that gets us to a working solution before we jump ahead to far more complex in the kernel models, like aware drivers that can toggle themselves between T=0/1.

Quoted 2 lines

Userspace might be able to bind a new driver that leaves the device in a lockable state on unbind, but that is not "just works" that is,

I wouldn't have the kernel leave the device in the locked state. That should always be userspace. The special driver may do whatever special setup is needed, then unbind and leave a normal unlocked device "prepped" for userspace locking without doing a FLR or something. Realistically I expect this to be a very rare requirement, I think this coming up just reflects the HW immaturity of some early TDISP devices.

Sensible mature devices should have no need of a pre-locking step. I think we should design toward that goal as the stable future and only try to enable a hacky work around for the problematic early devices. I certainly am not keen on seeing significant permanent kernel complexity to support this device design defect.

Quoted 5 lines

driver that expects the device arrives already running. Also, that main driver needs to be careful not to trigger typically benign actions like touch the command register to trip the device into ERROR state, or any device-specific actions that trip ERROR state but would otherwise be benign outside of TDISP."

As I said below, I disagree with this. You can't touch the *physical* command register but the cVM can certainly touch the *virtualized* command register. It up to the VMM To ensure this doesn't cause the device to fall out of RUN as part of virtualization.

I'd also say that the VMM should be responsible to set pBME=1 even if vBME=0? Shouldn't it? That simplifies even more things for the guest.

Quoted 7 lines
Quoted 4 lines

From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.

Agree, for PCI it would be simple to set a no-auto-probe policy for T=1 capable devices.

So then it is just a question of what does a userspace component need to do.

Quoted 5 lines

I do not want to burden the PCI core with TDISP compatibility hacks and workarounds if it turns out only a small handful of devices ever deploy a first generation TDISP Device Security Manager (DSM). L1 aiding L2, or TDISP simplicity improvements to allow the PCI core to handle this in a non-broken way, are what I expect if secure device assignment takes off.

Same feeling about pre-configuration :)

Quoted 5 lines
Quoted 3 lines

The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.

Do you agree that "device-specific-prep+lock" is the problem to solve?

Not "the" problem, but an design issue we need to accommodate but not endorse.

Quoted 9 lines
Quoted 4 lines

But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.

Hmm, having trouble not reading that back supporting my argument above:

Realistically few devices support TDISP lets require enhanced drivers to opt-into TDISP for the time being.

I would be comfortable if hitless RAS recovery for TDISP devices requires some kernel opt-in. But also I'm not sure how this should work from a security perspective. Should userspace also have to re-attest before allowing back to RUN? Clearly this is complicated.

Also, I would be comfortable to support this only for devices that do not require pre-configuration.

Jason

#179 lore raw
Quoted 11 lines · Jason Gunthorpe
Quoted 6 lines · On Fri, Aug 01, 2025 at 02:19:54PM -0700, [email protected] wrote:

On the host this establishes an SPDM session and sets up link encryption (IDE) with the physical device. Leave VMs out of the picture, this capability in isolation is a useful property. It addresses the similar threat model that Intel Total Memory Encryption (TME) or AMD Secure Memory Encryption (SME) go after, i.e. interposer on a physical link capturing data in flight.

Okay, maybe connect is not an intuitive name for opening IDE sessions..

Part of the rationale for a generic name is the TSM is free to assert that the link is secure without IDE. Think integrated devices where there is no expectation the link can be observed.

The host and guest side TSM operations are split into link/transport security and device/state security (private MMIO/DMA) concerns respectively. So maybe "secure_link" would be a better name for this host-side-only operation.

Quoted 8 lines
Quoted 3 lines

I started this project with "all existing T=0 drivers 'just work'" as a goal and a virtue. I have been begrudgingly pulled away from it from the slow drip of complexity it appears to push into the PCI core.

Do you have some examples? I don't really see what complexity there is if the solution it simply not auto bind any drivers to TDISP capable devices and userspace is responsible to manually bind a driver once it has reached T=1.

The example I have front of mind (confirmed by 2 vendors) is deferring the loading of guest-side device/state security capable firmware to the guest driver when the full device is assigned. In that scenario default device power-on firmware is capable of link/transport security, enough to get the device assigned. Guest needs to get the device/state security firmware loaded before TDISP state transitions are possible.

I do think RAS recovery needs it too, but like you say below that should come with conditions.

Quoted 11 lines

This seems like the minimum possible simplicitly for the kernel as simply everything is managed by userspace, and there is really no special kernel behavior beyond switching the DMA API of an unbound driver on the T=0/1 change.

Quoted 2 lines

The concern is neither userspace nor the PCI core have everything it needs to get the device to T=1.

Disagree, I think userspace can have everything. It may need some per-device userspace support in difficult cases, but userspace can deal with it..

I do think userspace can / must deal with it. Let me come back with actual patches and a sample test case. I see a potential path to support the above "prep" scenario without the mess of TDISP setup drivers, or the ugly complexity of driver toggles or a usermodehelper.

Quoted 8 lines
Quoted 2 lines

PCI core knows that the device is T=1 capable, but does not know how to preconfigure the device-specific lock state,

Userspace can do this. Can we define exactly what is needed to do this "pre-configure the device specific lock state"? At the very worst, for the most poorly designed device, userspace would have to bind a T=0 driver and then unbind it.

Quoted 4 lines

Again, I am trying to make something simple for the kernel that gets us to a working solution before we jump ahead to far more complex in the kernel models, like aware drivers that can toggle themselves between T=0/1.

Agree. When I talked about wishing for the simple TDISP case that is userspace can always "just lock" and "driver bind" without needing to worry about "prep", i.e any "prep" is always implied by "lock". That should be the baseline.

Quoted 11 lines
Quoted 2 lines

Userspace might be able to bind a new driver that leaves the device in a lockable state on unbind, but that is not "just works" that is,

I wouldn't have the kernel leave the device in the locked state. That should always be userspace. The special driver may do whatever special setup is needed, then unbind and leave a normal unlocked device "prepped" for userspace locking without doing a FLR or something. Realistically I expect this to be a very rare requirement, I think this coming up just reflects the HW immaturity of some early TDISP devices.

Quoted 5 lines

Sensible mature devices should have no need of a pre-locking step. I think we should design toward that goal as the stable future and only try to enable a hacky work around for the problematic early devices. I certainly am not keen on seeing significant permanent kernel complexity to support this device design defect.

Yeah, that is the nightmare I had last night. I completed the thought exercise about driver toggle and said, "whoops, nope, Jason is right, we can't design for that without leaving a permanent mess to cleanup". The end goal needs to look like straight line typical driver probe path for TDISP capable devices.

Quoted 6 lines
Quoted 5 lines

driver that expects the device arrives already running. Also, that main driver needs to be careful not to trigger typically benign actions like touch the command register to trip the device into ERROR state, or any device-specific actions that trip ERROR state but would otherwise be benign outside of TDISP."

Quoted 7 lines

As I said below, I disagree with this. You can't touch the *physical* command register but the cVM can certainly touch the *virtualized* command register. It up to the VMM To ensure this doesn't cause the device to fall out of RUN as part of virtualization.

I'd also say that the VMM should be responsible to set pBME=1 even if vBME=0? Shouldn't it? That simplifies even more things for the guest.

True. Although, now I am going back on my PCI core burden concern to wonder if *it* should handle a vBME on behalf of the driver if only because it may want to force the device out of the RUN state on driver unbind to meet typical pci_disable_device() expectations.

Alexey had this, I thought it was burdensome, now coming around.

Quoted 19 lines
Quoted 7 lines
Quoted 4 lines

From that principal the kernel should NOT auto probe drivers to T=0 devices that can be made T=1. Userspace should handle attaching HW to such devices, and userspace can sequence whatever is required, including the attestation and verifying.

Agree, for PCI it would be simple to set a no-auto-probe policy for T=1 capable devices.

So then it is just a question of what does a userspace component need to do.

Quoted 5 lines

I do not want to burden the PCI core with TDISP compatibility hacks and workarounds if it turns out only a small handful of devices ever deploy a first generation TDISP Device Security Manager (DSM). L1 aiding L2, or TDISP simplicity improvements to allow the PCI core to handle this in a non-broken way, are what I expect if secure device assignment takes off.

Same feeling about pre-configuration :)

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

The starting point must have the core code do this sequence for every driver. Once that is working we can talk about if other flows are needed.

Do you agree that "device-specific-prep+lock" is the problem to solve?

Not "the" problem, but an design issue we need to accommodate but not endorse.

I hear you, let me walk back from the cliff with patches.

Quoted 11 lines
Quoted 10 lines
Quoted 5 lines

But I think we can start with the idea that such RAS failures have to reload the driver too and work on improvements. Realistically few drivers have the sort of RAS features to consume this anyhow and maybe we introduce some "enhanced" driver mode to opt-into down the road.

Hmm, having trouble not reading that back supporting my argument above:

Realistically few devices support TDISP lets require enhanced drivers to opt-into TDISP for the time being.

Quoted 7 lines

I would be comfortable if hitless RAS recovery for TDISP devices requires some kernel opt-in. But also I'm not sure how this should work from a security perspective. Should userspace also have to re-attest before allowing back to RUN? Clearly this is complicated.

Also, I would be comfortable to support this only for devices that do not require pre-configuration.

That seems reasonable. You want hitless RAS? Give us hitless init.

#180 lore raw
Quoted 11 lines · On Sat, Aug 02, 2025 at 04:50:50PM -0700, [email protected] wrote:
Quoted 4 lines

Do you have some examples? I don't really see what complexity there is if the solution it simply not auto bind any drivers to TDISP capable devices and userspace is responsible to manually bind a driver once it has reached T=1.

The example I have front of mind (confirmed by 2 vendors) is deferring the loading of guest-side device/state security capable firmware to the guest driver when the full device is assigned. In that scenario default device power-on firmware is capable of link/transport security, enough to get the device assigned. Guest needs to get the device/state security firmware loaded before TDISP state transitions are possible.

Yeah, those are the only cases I know of too, and IMHO, they are just early devices. Clearly the clean answer is to put enough boot FW on the device's flash to get to T=1 mode, then have the trusted OS driver load the operating firmware from the trusted OS filesystem though the trusted bootloader T=1 device.

You effectively attest the bootloader, and then if you trust the bootloader you know that when the device gets to T=1 it can be trusted to properly run the FW the trusted driver provides.

Think about this more broadly, does the prep FW load idea make sense for something like SRIOV? No, it really doesn't. The hypervisor loaded FW that is running the PF should definately be strong enough to get to T=1 on the VM/VF side as well.

The non-SRIOV cases are quite often whole machine assignment scenarios. But I'm sensing alot of that space is moving toward bare metal machines instead of VMs.

I wonder if you can use all the CC machinery to attest and secure a bare metal host?

Quoted 2 lines

I do think RAS recovery needs it too, but like you say below that should come with conditions.

Especially RAS becomes simple because it basically follows the normal flows that existed prior to TDISP, with the exception of needing some attestation step.

I don't know alot about CC attestation, but maybe we can have userspace provide the kernel with the accepted measurement and then for RAS the kernel can FLR, remeasure and if the measurement is exactly the same go back into T=1 automatically as part of the PCI core FLR logic.

Quoted 4 lines

I do think userspace can / must deal with it. Let me come back with actual patches and a sample test case. I see a potential path to support the above "prep" scenario without the mess of TDISP setup drivers, or the ugly complexity of driver toggles or a usermodehelper.

I don't see how, something nasty has to be done in the kernel to allow an attached driver to switch between T=1 and T=0 "views" of the device and lockstep those changes with userspace. This is not so simple and it really basically exactly the same as driver binding.

I don't think we should be afraid of T=0 prep drivers in these early days.

Something more complex could come later if it is really warranted and people really insist on continuing this unclean device design strategy.

Quoted 5 lines

Yeah, that is the nightmare I had last night. I completed the thought exercise about driver toggle and said, "whoops, nope, Jason is right, we can't design for that without leaving a permanent mess to cleanup". The end goal needs to look like straight line typical driver probe path for TDISP capable devices.

Yeah, maybe it is worthwhile to someday try to figure out an alternative - keep in mind that critically this requires someone to also come with an intree driver that will use all these new APIs and capabilities!!!

So lets get walking first and then someone can come with some proposal, complete with a driver implementing it, and it can be judged. This project is already so big, and I'm pretty sure if you start to also need entirely new operating modes for drivers the basics will just get bogged down in that discussion, and very likely killed anyhow due to a lack of user.

Even if we decide that is prefered it is better to separate it and discuss it after the basics are merged. At least where I sit getting basic guest support is a big priority so I strongly want to strip it down to minimal as possible to make consistent progress steps.

Quoted 4 lines

True. Although, now I am going back on my PCI core burden concern to wonder if *it* should handle a vBME on behalf of the driver if only because it may want to force the device out of the RUN state on driver unbind to meet typical pci_disable_device() expectations.

Hiding some vBME in the PCI core might make sense if we can't get the VMM owners to agree to do it on the hypervisor side. It works better on the VMM side because there is always an IOMMU and the VMM can emulate BME by blocking DMA with the IOMMU.

But I would not allow/expect kernel device drivers to have anything to do with the TDISP states. Getting into RUN is fully sequenced by userspace, getting out of run should also be sequenced only by userspace.

Removing a driver does not change the trust state of the PCI device, so it shouldn't drop out of RUN. If userspace wishes to FLR the device after userspace asked to unbind it can, there are already sysfs controls for this IIRC.

Basically, all this says that Linux drivers that want to be used with T=1 should be well behaved, fully quite all their DMA on remove, and have no *functional* need for BME to do anyhting. We pretty much already expect this of drivers today, so I don't see an issue with strongly requiring it for T=1.

Keep in mind the flip side, almost no drivers are structured properly to forcibly quiet any DMA before pci_enable_device(). Some HW, like mlx5, can't do this at all without either using DMA to send a reset command or through FLR.

Quoted 9 lines
Quoted 7 lines

I would be comfortable if hitless RAS recovery for TDISP devices requires some kernel opt-in. But also I'm not sure how this should work from a security perspective. Should userspace also have to re-attest before allowing back to RUN? Clearly this is complicated.

Also, I would be comfortable to support this only for devices that do not require pre-configuration.

That seems reasonable. You want hitless RAS? Give us hitless init.

Yeah.. Realistically there are few drivers that can even do this today, mlx5 for example has such code (and it is hard!).

There is alot of investment required in the driver's core subsystem to make this work. netdev and RDMA can support a 'rebirth' sort of flow where the driver can disconnect the SW APIs, FLR the device, then reconnect in some way. However, for example, I recently had a discussion with DRM guys about RAS and they are not even doing the basic locking/etc to be able to do this. :\

Jason

#53 lore raw

Re: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl

In reply to Jason Gunthorpe
Quoted 7 lines · Jason Gunthorpe
Quoted 5 lines · Aneesh Kumar K.V (Arm)

With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent

Quoted 6 lines

I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?

But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??

I ended up exporting the guestmemfd's kvm_gmem_get_folio() for gfn->pfn and its fd a bit differently in iommufd - "no extra referencing":
https://github.com/AMDESE/linux-kvm/commit/f1ebd358327f026f413f8d3d64d46decfd6ab7f6

It is a new iommufd->kvm dependency though.

Quoted 4 lines

I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.

Yeah while doing the above, I was wondering if I want to pass the fd type when DMA-mapping from an fd or "detect" it as I do in the above commit or have some iommufd_fdmap_ops in this fd saying "(no) pinning needed" (or make this a flag of IOMMU_IOAS_MAP_FILE).

The "detection" is (mapping_inaccessible(mapping) && mapping_unevictable(mapping)), works for now.

btw in the AMD case, here it does not matter as much if it is private or shared, I map everything and let RMP and the VM deal with the permissions. Thanks,

Quoted 2 lines

Jason

Signature · 1 line
Alexey
author maintainer
#106 lore raw

Re: [RFC PATCH v1 14/38] coco: host: arm64: Device communication support

In reply to Jonathan Cameron
Quoted 202 lines · Jonathan Cameron
Quoted 45 lines · On Mon, 28 Jul 2025 19:21:51 +0530 "Aneesh Kumar K.V (Arm)"…

Add helpers for device communication from RMM

---
 arch/arm64/include/asm/rmi_cmds.h        |  11 ++
 arch/arm64/include/asm/rmi_smc.h         |  49 ++++++
 drivers/virt/coco/arm-cca-host/arm-cca.c |  45 ++++++
 drivers/virt/coco/arm-cca-host/rmm-da.c  | 198 +++++++++++++++++++++++
 drivers/virt/coco/arm-cca-host/rmm-da.h  |  41 +++++
 5 files changed, 344 insertions(+)

#endif /* __ASM_RMI_CMDS_H */

arch/arm64/include/asm/rmi_smc.h +25 -0
index a84ed61e5001..8bece465b670 100644
@@ -47,6 +47,7 @@
 
 
+#define RMI_DEV_COMM_EXIT_CACHE_REQ	BIT(0)
+#define RMI_DEV_COMM_EXIT_CACHE_RSP	BIT(1)
+#define RMI_DEV_COMM_EXIT_SEND		BIT(2)
+#define RMI_DEV_COMM_EXIT_WAIT		BIT(3)
+#define RMI_DEV_COMM_EXIT_MULTI		BIT(4)
+
+#define RMI_DEV_COMM_NONE	0
+#define RMI_DEV_COMM_RESPONSE	1
+#define RMI_DEV_COMM_ERROR	2
+
+#define RMI_PROTOCOL_SPDM		0
+#define RMI_PROTOCOL_SECURE_SPDM	1
+
+#define RMI_DEV_VCA			0
+#define RMI_DEV_CERTIFICATE		1
+#define RMI_DEV_MEASUREMENTS		2
+#define RMI_DEV_INTERFACE_REPORT	3
+
+struct rmi_dev_comm_enter {
+	u64 status;
+	u64 req_addr;
+	u64 resp_addr;
+	u64 resp_len;
+};
+

arch/arm64/include/asm/rmi_smc.h struct rmi_pdev_params { · line 376

Quoted 10 lines
arch/arm64/include/asm/rmi_smc.h +10 -0
+struct rmi_dev_comm_exit {
+	u64 flags;
+	u64 cache_req_offset;
+	u64 cache_req_len;
+	u64 cache_rsp_offset;
+	u64 cache_rsp_len;
+	u64 cache_obj_id;
+	u64 protocol;
+	u64 req_len;
+	u64 timeout;

In latest spec called rsp_timeout.
Not sure we care that much but if no strong reason otherwise, should aim to match the spec text. (Maybe this got renamed?)

Quoted 7 lines
arch/arm64/include/asm/rmi_smc.h +1 -0
+};
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
index 84d97dd41191..294a6ef60d5f 100644

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 92

Quoted 8 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5 -0
@@ -85,6 +85,45 @@ static void cca_tsm_pci_remove(struct pci_tsm *tsm)
 	vfree(dsc_pf0);
 }
 
+static int init_dev_communication_buffers(struct cca_host_comm_data *comm_data)
+{
+	int ret = -ENOMEM;
+
+	comm_data->io_params = (struct rmi_dev_comm_data *)get_zeroed_page(GFP_KERNEL);

Hmm. There isn't a DEFINE_FREE() yet for free_page(). Maybe time to add one. If we did then we'd use local variables until all allocations succeed then assign with no_free_ptr()

Quoted 5 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +5 -0
+	if (!comm_data->io_params)
+		goto err_out;
+
+	comm_data->resp_buff = (void *)__get_free_page(GFP_KERNEL);
+	if (!comm_data->resp_buff)

drivers/virt/coco/arm-cca-host/arm-cca.c static void cca_tsm_pci_remove(struct pci_tsm *tsm) · line 107

Quoted 10 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +10 -0
+		goto err_res_buff;
+
+	comm_data->req_buff = (void *)__get_free_page(GFP_KERNEL);
+	if (!comm_data->req_buff)
+		goto err_req_buff;
+
+
+	comm_data->io_params->enter.status = RMI_DEV_COMM_NONE;
+	comm_data->io_params->enter.resp_addr = virt_to_phys(comm_data->resp_buff);
+	comm_data->io_params->enter.req_addr  = virt_to_phys((void *)comm_data->req_buff);

I think it's already a a void * and even if it were some other pointer type no cast would be necessary.

Quoted 26 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +14 -0
+	comm_data->io_params->enter.resp_len = 0;
+
+	return 0;
+
+err_req_buff:
+	free_page((unsigned long)comm_data->resp_buff);
+err_res_buff:
+	free_page((unsigned long)comm_data->io_params);
+err_out:
+	return ret;
+}
+
 
+
 /* per root port unique with multiple restrictions. For now global */
 static DECLARE_BITMAP(cca_stream_ids, MAX_STREAM_ID);
@@ -124,6 +163,7 @@ static int cca_tsm_connect(struct pci_dev *pdev)
 	rc = tsm_ide_stream_register(pdev, ide);
 	if (rc)
 		goto err_tsm;
+	init_dev_communication_buffers(&dsc_pf0->comm_data);
 	/*
 	 * Take a module reference so that we won't call unregister
 	 * without rme_unasign_device

drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 175

Quoted 3 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +0 -0
@@ -133,6 +173,11 @@ static int cca_tsm_connect(struct pci_dev *pdev)
 		goto err_tsm;
 	}
 	rme_asign_device(pdev);

rme_assign_device() - I obviously missed this earlier!

drivers/virt/coco/arm-cca-host/arm-cca.c static int cca_tsm_connect(struct pci_dev *pdev) · line 177

Quoted 2 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +2 -0
+	/*
+	 * Schedule a work to fetch device certificate and setup IDE

Single line comment probably fine here. Though it perhaps doesn't add much over the function name.

Quoted 16 lines
drivers/virt/coco/arm-cca-host/arm-cca.c +3 -0
+	 */
+	schedule_rme_ide_setup(pdev);
+
 	/*
 	 * Once ide is setup enable the stream at endpoint
 	 * Root port will be done by RMM
drivers/virt/coco/arm-cca-host/rmm-da.c +2 -0
index 426e530ac182..d123940ce82e 100644
@@ -148,3 +148,201 @@ int rme_asign_device(struct pci_dev *pci_dev)
 err_out:
 	return ret;
 }
+
+static int doe_send_req_resp(struct pci_tsm *tsm)

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 162

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+{
+	u8 protocol;
+	int ret, data_obj_type;
+	struct cca_host_comm_data *comm_data;
+	struct rmi_dev_comm_exit *io_exit;
+
+	comm_data = to_cca_comm_data(tsm->pdev);
+
+	io_exit = &comm_data->io_params->exit;
+	protocol = io_exit->protocol;

For all these I'd combine with the declarations.

Quoted 24 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +24 -0
+
+	pr_debug("doe_req size:%lld doe_io_type=%d\n", io_exit->req_len, (int)protocol);
+
+	if (protocol == RMI_PROTOCOL_SPDM)
+		data_obj_type = PCI_DOE_PROTO_CMA;
+	else if (protocol == RMI_PROTOCOL_SECURE_SPDM)
+		data_obj_type = PCI_DOE_PROTO_SSESSION;
+	else
+		return -EINVAL;
+
+	ret = pci_tsm_doe_transfer(tsm->dsm_dev, data_obj_type,
+				   comm_data->req_buff, io_exit->req_len,
+				   comm_data->resp_buff, PAGE_SIZE);
+	pr_debug("doe returned:%d\n", ret);
+	return ret;
+}
+
+/* Parallel update for cca_dsc contents FIXME!! */
+static int __do_dev_communicate(int type, struct pci_tsm *tsm)
+{
+	int ret;
+	bool is_multi;
+	u8 *cache_buf;
+	int *cache_offset;

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 196

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+	int nbytes, cache_remaining;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct rmi_dev_comm_exit *io_exit;
+	struct rmi_dev_comm_enter *io_enter;
+	struct cca_host_comm_data *comm_data;
+
+
+	comm_data = to_cca_comm_data(tsm->pdev);
+	io_enter = &comm_data->io_params->enter;
+	io_exit = &comm_data->io_params->exit;

Might as well set these local variables as the declaration point above. None of them will be very long lines.

Quoted 15 lines

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 206

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->dsm_dev);
+redo_communicate:
+	is_multi = false;
+
+	if (type == PDEV_COMMUNICATE)
+		ret = rmi_pdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
+					   virt_to_phys(comm_data->io_params));
+	else
+		ret = RMI_ERROR_INPUT;

I'd split this case out and return here farther than using the match below as it feels like the error message auto to be more specific. Something about type not matching.

This get updated in the follow up patch as below.

	if (type == PDEV_COMMUNICATE)
		ret = rmi_pdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
					   virt_to_phys(comm_data->io_params));
	else {
		struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi);

		ret = rmi_vdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
					   virt_to_phys(host_tdi->rmm_vdev),
					   virt_to_phys(comm_data->io_params));
	}
Quoted 105 lines
Quoted 32 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31 -0
+	if (ret != RMI_SUCCESS) {
+		pr_err("pdev communicate error\n");
+		return ret;
+	}
+
+	/* caching request from RMM */
+	if (io_exit->flags & RMI_DEV_COMM_EXIT_CACHE_RSP) {
+		switch (io_exit->cache_obj_id) {
+		case RMI_DEV_VCA:
+			cache_buf = dsc_pf0->vca.buf;
+			cache_offset = &dsc_pf0->vca.size;
+			cache_remaining = sizeof(dsc_pf0->vca.buf) - *cache_offset;
+			break;
+		case RMI_DEV_CERTIFICATE:
+			cache_buf = dsc_pf0->cert_chain.cache.buf;
+			cache_offset = &dsc_pf0->cert_chain.cache.size;
+			cache_remaining = sizeof(dsc_pf0->cert_chain.cache.buf) - *cache_offset;
+			break;
+		default:
+			/* FIXME!! depending on the DevComms status,
+			 * it might require to ABORT the communcation.
+			 */
+			return -EINVAL;
+		}
+
+		if (io_exit->cache_rsp_len > cache_remaining)
+			return -EINVAL;
+
+		memcpy(cache_buf + *cache_offset,
+		       (comm_data->resp_buff + io_exit->cache_rsp_offset), io_exit->cache_rsp_len);
+		*cache_offset += io_exit->cache_rsp_len;

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 247

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+	}
+
+	/*
+	 * wait for last packet request from RMM.
+	 * We should not find this because our device communication in synchronous
+	 */
+	if (io_exit->flags & RMI_DEV_COMM_EXIT_WAIT)
+		return -ENXIO;
+
+	is_multi = !!(io_exit->flags & RMI_DEV_COMM_EXIT_MULTI);

!! doesn't add anything here that I can see over

	is_multi = io_exit->flags & RMI_DEV_COMM_EXIT_MULTI;
Quoted 31 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +31 -0
+
+	/* next packet to send */
+	if (io_exit->flags & RMI_DEV_COMM_EXIT_SEND) {
+		nbytes = doe_send_req_resp(tsm);
+		if (nbytes < 0) {
+			/* report error back to RMM */
+			io_enter->status = RMI_DEV_COMM_ERROR;
+		} else {
+			/* send response back to RMM */
+			io_enter->resp_len = nbytes;
+			io_enter->status = RMI_DEV_COMM_RESPONSE;
+		}
+	} else {
+		/* no data transmitted => no data received */
+		io_enter->resp_len = 0;
+	}
+
+	/* The call need to do multiple request/respnse */
+	if (is_multi)
+		goto redo_communicate;
+
+	return 0;
+}
+
+static int do_dev_communicate(int type, struct pci_tsm *tsm, int target_state)
+{
+	int ret;
+	unsigned long state;
+	unsigned long error_state;
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	struct rmi_dev_comm_enter *io_enter;

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 288

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->dsm_dev);
+	io_enter = &dsc_pf0->comm_data.io_params->enter;
+	io_enter->resp_len = 0;
+	io_enter->status = RMI_DEV_COMM_NONE;
+
+	state = -1;
+	do {
+		ret = __do_dev_communicate(type, tsm);
+		if (ret != 0) {

if (ret)

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 290

Quoted 2 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +2 -0
+			pr_err("dev communication error\n");
+			break;

I'd just return in error cases.

Quoted 9 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +9 -0
+		}
+
+		if (type == PDEV_COMMUNICATE) {
+			ret = rmi_pdev_get_state(virt_to_phys(dsc_pf0->rmm_pdev),
+						 &state);
+			error_state = RMI_PDEV_ERROR;
+		}
+		if (ret != 0) {
+			pr_err("Get dev state error\n");
Quoted 8 lines

drivers/virt/coco/arm-cca-host/rmm-da.c int rme_asign_device(struct pci_dev *pci_dev) · line 307

Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +8 -0
+			break;
+		}
+	} while (state != target_state && state != error_state);
+
+	pr_info("dev_io_complete: status: %d state:%ld\n", ret, state);
+
+	return state;
+}

Thanks for the review comments. I'll update the patch with the suggested changes.

-aneesh

author maintainer
#110 lore raw

Re: [RFC PATCH v1 15/38] coco: host: arm64: Stop and destroy the physical device

In reply to Jonathan Cameron
Quoted 10 lines · Jonathan Cameron
Quoted 1 line · On Mon, 28 Jul 2025 19:21:52 +0530 "Aneesh Kumar K.V (Arm)"…

Add support for stopping and destroying physical devices.

I think it's an odd mix to not do create and destroy in a single patch. Same with start and stop.
Leaves reviewers thinking perhaps you weren't cleaning up properly an any error paths are much less obvious.

I’ll fold this into the patch that adds the pdev_create functionality.

-aneesh

author maintainer
#117 lore raw

Re: [RFC PATCH v1 19/38] coco: host: arm64: set_pubkey support

In reply to Jonathan Cameron
Quoted 170 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:56 +0530 "Aneesh Kumar K.V (Arm)"…

Add changes to share the device's public key with the RMM.

A few minor comments inline.

Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +15 -0
index ec8c5bfcee35..3715e6d58c83 100644
@@ -6,6 +6,9 @@
 #include <linux/pci.h>
 #include <linux/pci-ecam.h>
 #include <asm/rmi_cmds.h>
+#include <crypto/internal/rsa.h>
+#include <keys/asymmetric-type.h>
+#include <keys/x509-parser.h>
 
 #include "rmm-da.h"
 
@@ -311,6 +314,136 @@ static int do_pdev_communicate(struct pci_tsm *tsm, int target_state)
 	return do_dev_communicate(PDEV_COMMUNICATE, tsm, target_state);
 }
 
+static int parse_certificate_chain(struct pci_tsm *tsm)
+{
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	unsigned int chain_size;
+	unsigned int offset = 0;
+	u8 *chain_data;
+	int ret = 0;
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->pdev);
+	chain_size = dsc_pf0->cert_chain.cache.size;
+	chain_data = dsc_pf0->cert_chain.cache.buf;
+

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 338

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+	while (offset < chain_size) {
+		unsigned int cert_len =
+			x509_get_certificate_length(chain_data + offset,
+						    chain_size - offset);
+		struct x509_certificate *cert =
+			x509_cert_parse(chain_data + offset, cert_len);
+
+		if (IS_ERR(cert)) {
+			pr_warn("%s(): parsing of certificate chain not successful\n", __func__);
+			ret = PTR_ERR(cert);

Direct return looks fine here. Maybe add a DEFINE_FREE(x509_cert,...) as then can use direct returns throughout.

Quoted 23 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +23 -0
+			break;
+		}
+
+		if (offset + cert_len == chain_size) {
+			dsc_pf0->cert_chain.public_key = kzalloc(cert->pub->keylen, GFP_KERNEL);
+			if (!dsc_pf0->cert_chain.public_key) {
+				ret = -ENOMEM;
+				x509_free_certificate(cert);
+				break;
+			}
+
+			if (!strcmp("ecdsa-nist-p256", cert->pub->pkey_algo)) {
+				dsc_pf0->rmi_signature_algorithm = RMI_SIG_ECDSA_P256;
+			} else if (!strcmp("ecdsa-nist-p384", cert->pub->pkey_algo)) {
+				dsc_pf0->rmi_signature_algorithm = RMI_SIG_ECDSA_P384;
+			} else if (!strcmp("rsa", cert->pub->pkey_algo)) {
+				dsc_pf0->rmi_signature_algorithm = RMI_SIG_RSASSA_3072;
+			} else {
+				ret = -ENXIO;
+				x509_free_certificate(cert);
+				break;
+			}
+			memcpy(dsc_pf0->cert_chain.public_key, cert->pub->key, cert->pub->keylen);

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 371

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+			dsc_pf0->cert_chain.public_key_size = cert->pub->keylen;
+		}
+
+		x509_free_certificate(cert);
+
+		offset += cert_len;
+	}
+
+	if (ret == 0)
+		dsc_pf0->cert_chain.valid = true;
	if (ret)
		return ret;

	dsc_pf0->cert_chain.valid = true;

	return 0;

would be my preference for style here but others may disagree.

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 379

Quoted 8 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +8 -0
+
+	return ret;
+}
+
+static int pdev_set_public_key(struct pci_tsm *tsm)
+{
+	struct rmi_public_key_params *key_shared;
+	unsigned long expected_key_len = 0;

Don't set this. It's only used in places where it is explicitly set and if it is used anywhere else we want the compiler to tell us.

Quoted 30 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +30 -0
+	struct cca_host_dsc_pf0 *dsc_pf0;
+	int ret;
+
+	dsc_pf0 = to_cca_dsc_pf0(tsm->pdev);
+	/* Check that all the necessary information was captured from communication */
+	if (!dsc_pf0->cert_chain.valid)
+		return -EINVAL;
+
+	key_shared = (struct rmi_public_key_params *)get_zeroed_page(GFP_KERNEL);
+	if (!key_shared)
+		return -ENOMEM;
+
+	key_shared->rmi_signature_algorithm = dsc_pf0->rmi_signature_algorithm;
+
+	switch (key_shared->rmi_signature_algorithm) {
+	case RMI_SIG_ECDSA_P384:
+		expected_key_len = 97;
+
+		if (dsc_pf0->cert_chain.public_key_size != expected_key_len)
+			return -EINVAL;
+		key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size;
+		memcpy(key_shared->public_key,
+		       dsc_pf0->cert_chain.public_key,
+		       dsc_pf0->cert_chain.public_key_size);
+		key_shared->metadata_len = 0;
+		break;
+	case RMI_SIG_ECDSA_P256:
+		expected_key_len = 65;
+
+		if (dsc_pf0->cert_chain.public_key_size != expected_key_len)

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 419

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+			return -EINVAL;
+		key_shared->public_key_len = dsc_pf0->cert_chain.public_key_size;
+		memcpy(key_shared->public_key,
+		       dsc_pf0->cert_chain.public_key,
+		       dsc_pf0->cert_chain.public_key_size);
+		key_shared->metadata_len = 0;
+		break;
+	case RMI_SIG_RSASSA_3072:
+		expected_key_len = 385;
+		struct rsa_key rsa_key = {0};

Shouldn't define this inline. Maybe move up a line and add some {} to set the scope to this case statement.

Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +5 -0
+		int ret_rsa_parse = rsa_parse_pub_key(&rsa_key,
+						      dsc_pf0->cert_chain.public_key,
+						      dsc_pf0->cert_chain.public_key_size);
+		/* This also checks the key_len */
+		if (ret_rsa_parse)

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 434

Quoted 10 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +10 -0
+			return ret_rsa_parse;
+		/*
+		 * exponent is usally 65537 (size = 24bits) but in rare cases
+		 * it size can be as large as the modulus
+		 */
+		if (rsa_key.e_sz > expected_key_len)
+			return -EINVAL;
+		key_shared->public_key_len = rsa_key.n_sz;
+		key_shared->metadata_len = rsa_key.e_sz;
+		memcpy(key_shared->public_key, (unsigned char *)rsa_key.n, rsa_key.n_sz);

Why is the cast needed?

Quoted 11 lines

drivers/virt/coco/arm-cca-host/rmm-da.c static int do_pdev_communicate(struct pci_tsm *tsm, int target_state) · line 445

Quoted 11 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +11 -0
+		memcpy(key_shared->metadata, (unsigned char *)rsa_key.e, rsa_key.e_sz);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = rmi_pdev_set_pubkey(virt_to_phys(dsc_pf0->rmm_pdev),
+				  virt_to_phys(key_shared));
+	free_page((unsigned long)key_shared);
+	return ret;
+}

Thanks for the review comments. I'll update the patch with the suggested changes.

-aneesh

author maintainer
#123 lore raw

Re: [RFC PATCH v1 21/38] coco: host: arm64: Add support for virtual device communication

In reply to Jonathan Cameron
Quoted 9 lines · Jonathan Cameron
Quoted 3 lines · On Mon, 28 Jul 2025 19:21:58 +0530 "Aneesh Kumar K.V (Arm)"…

Add support for vdev_communicate with RMM.

One minor comment.

Quoted 8 lines

drivers/virt/coco/arm-cca-host/rmm-da.c static int __do_dev_communicate(int type, struct pci_tsm *tsm)

Quoted 5 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +0 -0
index 41314db1d568..8635f361bbe8 100644
@@ -207,8 +207,14 @@ static int __do_dev_communicate(int type, struct pci_tsm *tsm)

Can type parameter be an enum so we can see all cases are covered?

Sure, will update the patch

Quoted 13 lines

drivers/virt/coco/arm-cca-host/rmm-da.c static int __do_dev_communicate(int type, struct pci_tsm *tsm) · line 216

Quoted 13 lines
drivers/virt/coco/arm-cca-host/rmm-da.c +7 -2
 	if (type == PDEV_COMMUNICATE)
 		ret = rmi_pdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
 					   virt_to_phys(comm_data->io_params));
-	else
-		ret = RMI_ERROR_INPUT;
+	else {
+		struct cca_host_tdi *host_tdi = container_of(tsm->tdi, struct cca_host_tdi, tdi);
+
+		ret = rmi_vdev_communicate(virt_to_phys(dsc_pf0->rmm_pdev),
+					   virt_to_phys(host_tdi->rmm_vdev),
+					   virt_to_phys(comm_data->io_params));
+	}

-aneesh

author maintainer
#132 lore raw

Re: [RFC PATCH v1 24/38] arm64: CCA: Register guest tsm callback

In reply to Jonathan Cameron
Quoted 56 lines · Jonathan Cameron
Quoted 6 lines · On Mon, 28 Jul 2025 19:22:01 +0530 "Aneesh Kumar K.V (Arm)"…

Register the TSM callback if the DA feature is supported by RSI.

Additionally, adjust the build order so that the TSM class is created before the arm-cca-guest driver initialization.

See below.

Quoted 13 lines
arch/arm64/kernel/rsi.c +1 -0
index bf9ea99e2aa1..ef06c083990a 100644
@@ -15,6 +15,7 @@
 #include <asm/rsi.h>
 
 static struct realm_config config;
+static unsigned long rsi_feat_reg0;
 
 unsigned long prot_ns_shared;
 EXPORT_SYMBOL(prot_ns_shared);

arch/arm64/kernel/rsi.c EXPORT_SYMBOL(prot_ns_shared); · line 28

Quoted 6 lines
arch/arm64/kernel/rsi.c +3 -0
@@ -22,6 +23,12 @@ EXPORT_SYMBOL(prot_ns_shared);
 DEFINE_STATIC_KEY_FALSE_RO(rsi_present);
 EXPORT_SYMBOL(rsi_present);
 
+bool rsi_has_da_feature(void)
+{
+	return !!u64_get_bits(rsi_feat_reg0, RSI_FEATURE_REGISTER_0_DA);

!! not needed.

Quoted 23 lines
arch/arm64/kernel/rsi.c +2 -0
+}
+EXPORT_SYMBOL_GPL(rsi_has_da_feature);
 
 
drivers/virt/coco/arm-cca-guest/arm-cca.c +3 -1
index 547fc2c79f7d..3adbbd67e06e 100644
@@ -1,6 +1,6 @@
 
 static int cca_guest_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -200,11 +256,22 @@ static int cca_guest_probe(struct platform_device *pdev)
 		return -ENODEV;
 
 	ret = tsm_report_register(&arm_cca_tsm_report_ops, NULL);
-	if (ret < 0)
+	if (ret < 0) {
 		pr_err("Error %d registering with TSM\n", ret);
+		goto err_out;
+	}
 
Quoted 12 lines

drivers/virt/coco/arm-cca-guest/arm-cca.c static int cca_guest_probe(struct platform_device *pdev) · line 272

Quoted 9 lines
drivers/virt/coco/arm-cca-guest/arm-cca.c +8 -0
 	ret = devm_add_action_or_reset(&pdev->dev, unregister_cca_tsm_report, NULL);
+	if (ret < 0) {
+		pr_err("Error %d registering devm action\n", ret);
+		unregister_cca_tsm_report(NULL);
+		goto err_out;
+	}
+
+	if (rsi_has_da_feature())
+		ret = cca_tsm_register(pdev);

Why do we not need to call unregister_cca_tsm_report() if this fails?

tsm_report and DA are independent functionalities. I’ll update the guest probe to return success so that tsm_report remains available even if the DA tsm registration fails.

-aneesh

author maintainer
#152 lore raw

Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report

In reply to Arto Merilainen
Quoted 27 lines · Arto Merilainen
Quoted 6 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+	for (int i = 0; i < interface_report->mmio_range_count; i++, mmio_range++) {
+
+		/*FIXME!! units in 4K size*/
+		range_id = FIELD_GET(TSM_INTF_REPORT_MMIO_RANGE_ID, mmio_range->range_attributes);
+
+		/* no secure interrupts */

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 297

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+		if (msix_tbl_bar != -1 && range_id == msix_tbl_bar) {
+			pr_info("Skipping misx table\n");
+			continue;
+		}
+
+		if (msix_pba_bar != -1 && range_id == msix_pba_bar) {
+			pr_info("Skipping misx pba\n");
+			continue;
+		}
+

MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated. It is enough to have sufficient isolation within the BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.

Quoted 4 lines

Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?

Yes, that was added because at one point the FVP model was including the MSI-X table and PBA regions in the interface report.

If I understand correctly, we shouldn't expect to see those regions in the report unless secure interrupts are supported. The BAR-based skipping was added as a workaround to handle the FVP issue.

I believe we can drop that workaround now—if those regions are incorrectly present, the below validation logic should catch and reject them appropriately. Does that sound reasonable?

		/* No secure interrupts, we should not find this set, ignore for now. */
		if (FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes) ||
		    FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes)) {
			pci_info(pdev, "Skipping MSIX (%ld/%ld) range [%d] #%d %d pages, %llx..%llx\n",
				 FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes),
				 FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes),
				 i, range_id, mmio_range->num_pages, r->start, r->end);
			continue;
		}

-aneesh

author maintainer
#171 lore raw

Re: [RFC PATCH v1 38/38] coco: guest: arm64: Add support for fetching device info

In reply to Jonathan Cameron
Quoted 3 lines · Jonathan Cameron

On Mon, 28 Jul 2025 19:22:15 +0530
"Aneesh Kumar K.V (Arm)" <[email protected]> wrote:

...

Quoted 22 lines
Quoted 13 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +4 -0
+	return ret;
+}
+
 int rsi_device_lock(struct pci_dev *pdev)
 {
 	unsigned long ret;
 	unsigned long tdisp_version;
 	struct rsi_device_measurements_params *rsi_dev_meas;
+	struct rsi_device_info *dev_info;
 	struct cca_guest_dsc *dsm = to_cca_guest_dsc(pdev);
 	int vdev_id = (pci_domain_nr(pdev->bus) << 16) |
 		PCI_DEVID(pdev->bus->number, pdev->devfn);

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 348

Quoted 5 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +2 -0
@@ -252,8 +344,44 @@ int rsi_device_lock(struct pci_dev *pdev)
 		return -EIO;
 	}
 
+	/* RMM expects sizeof(dev_info) (512 bytes) aligned address */
+	dev_info = kmalloc(sizeof(*dev_info), GFP_KERNEL);

Use a __free(kfree) here (and direct returns on errors) given it's freed in all paths and we don't care if it is freed before or after verifying the digests.

Quoted 5 lines

I'm being slow today, but what in that enforces the alignment? I guess it's that the structure happens to be big enough that it happens naturally?

I'd allocate max(512, sizeof(*dev_info)) to make it explicitly the case.

yes, struct rsi_device_info is 512 bytes.

Quoted 15 lines
Quoted 15 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +14 -0
+	if (!dev_info) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	ret = rsi_rdev_get_info(vdev_id, dsm->instance_id, virt_to_phys(dev_info));
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to get device digests (%lu)\n", ret);
+		ret = -EIO;
+		kfree(dev_info);
+		goto err_out;
+	}
+
+	dsm->dev_info.attest_type   = dev_info->attest_type;
Quoted 9 lines

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 368

Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+	dsm->dev_info.cert_id       = dev_info->cert_id;
+	dsm->dev_info.hash_algo     = dev_info->hash_algo;
+	memcpy(dsm->dev_info.cert_digest, dev_info->cert_digest, SHA512_DIGEST_SIZE);
+	memcpy(dsm->dev_info.meas_digest, dev_info->meas_digest, SHA512_DIGEST_SIZE);
+	memcpy(dsm->dev_info.report_digest, dev_info->report_digest, SHA512_DIGEST_SIZE);
+

Can't you memcpy the whole thing in one go?

yes. But won't that be confusing? Is there a difference? Also struct dsm_device_info is not same as struct rsi_device_info. We don't need to keep all that padding in dsm_device_info.

Quoted 65 lines
Quoted 4 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +3 -0
+	kfree(dev_info);
+	/*
+	 * Verify that the digests of the provided reports match with the

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 381

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+	 * digests from RMM
+	 */
+	ret = verify_digests(dsm);
+	if (ret) {
+		pci_err(pdev, "device digest validation failed (%ld)\n", ret);
+		return ret;
+	}
+
+	return 0;
+err_out:

I'll always grumble about these. To me it always makes the code less readable. Some others disagree though ;(

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 384

Quoted 3 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +1 -0
 	return ret;
 }
+

Looks like this should have been in an earlier patch.

Quoted 19 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +0 -0
 static inline unsigned long rsi_rdev_start(struct pci_dev *pdev,
 					   unsigned long vdev_id, unsigned long inst_id)
 {
drivers/virt/coco/arm-cca-guest/rsi-da.h +1 -0
index f26156d9be81..e8953b8e85a3 100644
@@ -10,6 +10,7 @@
 #include <linux/pci-tsm.h>
 #include <asm/rsi_smc.h>
 #include <asm/rhi.h>
+#include <crypto/sha2.h>
 
 struct pci_tdisp_device_interface_report {
 	u16 interface_info;
@@ -33,6 +34,17 @@ struct pci_tdisp_mmio_range {
 #define TSM_INTF_REPORT_MMIO_RESERVED		GENMASK(15, 4)
 #define TSM_INTF_REPORT_MMIO_RANGE_ID		GENMASK(31, 16)
 

drivers/virt/coco/arm-cca-guest/rsi-da.h struct pci_tdisp_mmio_range { · line 46

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +10 -0
+struct dsm_device_info {
+	u64 flags;
+	u64 attest_type;
+	u64 cert_id;
+	u64 hash_algo;
+	u8 cert_digest[SHA512_DIGEST_SIZE];
+	u8 meas_digest[SHA512_DIGEST_SIZE];
+	u8 report_digest[SHA512_DIGEST_SIZE];
+};
+

One probably enough.

Quoted 11 lines
drivers/virt/coco/arm-cca-guest/rsi-da.h +2 -0
+
 struct cca_guest_dsc {
 	struct pci_tsm_pf0 pci;
 	unsigned long instance_id;
@@ -42,6 +54,7 @@ struct cca_guest_dsc {
 	int certificate_size;
 	void *measurements;
 	int measurements_size;
+	struct dsm_device_info dev_info;
 };
 
Quoted 1 line

drivers/virt/coco/arm-cca-guest/rsi-da.h struct cca_guest_dsc { · line 60

Quoted 1 line
drivers/virt/coco/arm-cca-guest/rsi-da.h +0 -0
 static inline struct cca_guest_dsc *to_cca_guest_dsc(struct pci_dev *pdev)

-aneesh

author maintainer
#18 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Jason Gunthorpe
Quoted 72 lines · Jason Gunthorpe
Quoted 51 lines · Aneesh Kumar K.V
Quoted 49 lines · Jason Gunthorpe
Quoted 33 lines · Aneesh Kumar K.V
Quoted 31 lines · Jason Gunthorpe

arch/arm64/mm/mem_encrypt.c int set_memory_decrypted(unsigned long addr, int numpages) · line 56

Quoted 9 lines · Aneesh Kumar K.V (Arm)
arch/arm64/mm/mem_encrypt.c +5 -0
@@ -48,3 +49,12 @@ int set_memory_decrypted(unsigned long addr, int numpages)
 	return crypt_ops->decrypt(addr, numpages);
 }
 EXPORT_SYMBOL_GPL(set_memory_decrypted);
+
+bool force_dma_unencrypted(struct device *dev)
+{
+	if (dev->tdi_enabled)
+		return false;

Is this OK? I see code like this:

static inline dma_addr_t phys_to_dma_direct(struct device *dev,
phys_addr_t phys)

{
	if (force_dma_unencrypted(dev))
		return phys_to_dma_unencrypted(dev, phys);
	return phys_to_dma(dev, phys);

What are the ARM rules for generating dma addreses?

1) Device is T=0, memory is unencrypted, call dma_addr_unencrypted() and do "top bit IBA set"

2) Device is T=1, memory is encrypted, use the phys_to_dma() normally

3) Device it T=1, memory is uncrypted, use the phys_to_dma() normally??? Seems odd, I would have guessed the DMA address sould be the same as case #1?

Quoted 5 lines
Quoted 2 lines

Can you document this in a comment?

If a device is operating in secure mode (T=1), it is currently assumed that only access to private (encrypted) memory is supported.

No, this is no how the PCI specs were written as far as I understand. The XT bit thing is supposed to add more fine grained device side control over what memory the DMA can target. T alone does not do that.

Quoted 3 lines

It is unclear whether devices would need to perform DMA to shared (unencrypted) memory while operating in this mode, as TLPs with T=1 are generally expected to target private memory.

Quoted 8 lines
Quoted 2 lines

PCI SIG supports it, kernel should support it.

Would we also need a separate DMA allocation API for allocating addresses intended to be shared with the non-secure hypervisor?

Are there any existing drivers in the kernel that already require such shared allocations, which I could use as a reference?

The most likely case in the near term is PCI P2P to shared MMIO.

I don't know any way to allocate shared memory in a driver??

At the bare minimum this patch should be documenting the correct architecture and outlining any gaps in the current implementation.

I also don't really understand what the above code is even doing.. Isn't the design on ARM that the IPA always encodes the shared/private in the top bit?

Quoted 6 lines

How do we get a shared page that does not already have a phys_addr_t in the shared IPA? Shouldn't the kernel have switched to the shared IPA alias when it returned the swiotlb buffer? eg why do we need to do:

#define dma_addr_unencrypted(x) ((x) | PROT_NS_SHARED)

swiotlb virt addr is updated in the direct map page table such that we have the correct attribute set. For ex: swiotlb_update_mem_attributes uses set_memory_decrypted() to mark the memory as shared.

	set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);

However, when mapping swiotlb regions to obtain a dma_addr_t, we still need to explicitly convert the physical address:

swiotlb_map()

	swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
        ...

	/* Ensure that the address returned is DMA'ble */
	dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);

Note that we don’t update the phys_addr_t to set the top bit. For reference:

	tlb_addr = slot_addr(pool->start, index) + offset;

-aneesh

#153 lore raw

Re: [RFC PATCH v1 34/38] coco: guest: arm64: Validate mmio range found in the interface report

In reply to Aneesh Kumar K.V
Quoted 29 lines · Aneesh Kumar K.V
Quoted 27 lines · Arto Merilainen
Quoted 6 lines · Aneesh Kumar K.V (Arm)
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+    for (int i = 0; i < interface_report->mmio_range_count; i++, mmio_range++) {
+
+            /*FIXME!! units in 4K size*/
+            range_id = FIELD_GET(TSM_INTF_REPORT_MMIO_RANGE_ID, mmio_range->range_attributes);
+
+            /* no secure interrupts */

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 297

Quoted 10 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +10 -0
+            if (msix_tbl_bar != -1 && range_id == msix_tbl_bar) {
+                    pr_info("Skipping misx table\n");
+                    continue;
+            }
+
+            if (msix_pba_bar != -1 && range_id == msix_pba_bar) {
+                    pr_info("Skipping misx pba\n");
+                    continue;
+            }
+

MSI-X and PBA can be placed to a BAR that has other registers as well. While the PCIe specification recommends BAR-level isolation for MSI-X structures, it is not mandated. It is enough to have sufficient isolation within the BAR. Therefore, skipping the MSI-X and PBA BARs altogether may leave registers unintentionally mapped via unprotected IPA when they should have been mapped via protected IPA.

Quoted 7 lines
Quoted 4 lines

Instead of skipping the whole BAR, would it make sense to determine where the MSI-X related regions reside, and skip validation only from these regions?

Yes, that was added because at one point the FVP model was including the MSI-X table and PBA regions in the interface report.

The issue I am raising is a different one. The MSI-X table and PBA may reside in the middle of a BAR range, and the BAR range may contain registers which should be accessible only via protected IPA. In this case I would expect the pages before and after the MSI-X table (and PBA) to be validated while the pages related to MSI-X structures would be left unprotected.

Given that the MSI-X table or PBA regions shouldn't be present in the interface report, the code needs to first find out where the MSI-X structures reside, and use this information to determine which "sub-ranges" should be skipped over during validation.

Quoted 4 lines

If I understand correctly, we shouldn't expect to see those regions in the report unless secure interrupts are supported. The BAR-based skipping was added as a workaround to handle the FVP issue.

Correct. Assuming that RMM doesn't lock the MSI-X table, I'd expect these regions to be omitted in the interface report.

Quoted 4 lines

I believe we can drop that workaround now—if those regions are incorrectly present, the below validation logic should catch and reject them appropriately. Does that sound reasonable?

Quoted 11 lines
                 /* No secure interrupts, we should not find this set, ignore for now. */
                 if (FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes) ||
                     FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes)) {
                         pci_info(pdev, "Skipping MSIX (%ld/%ld) range [%d] #%d %d pages, %llx..%llx\n",
                                  FIELD_GET(TSM_INTF_REPORT_MMIO_MSIX_TABLE, mmio_range->range_attributes),
                                  FIELD_GET(TSM_INTF_REPORT_MMIO_PBA, mmio_range->range_attributes),
                                  i, range_id, mmio_range->num_pages, r->start, r->end);
                         continue;
                 }

First, I think this is skipping over the whole range => if there are registers outside of the MSI-X structures (but within the same BAR), they won't be validated.

Second, if the MSI-X regions are present in the interface report, wouldn't it - in the common case - mean that the device expects the ranges to be accessed with T=1? If this happens unexpectedly, it sounds that MSI-X wouldn't be usable. I wonder if the code should simply return an error instead if informing the user via pci_info()...

- R2

author maintainer
#54 lore raw

Re: [RFC PATCH v1 10/38] iommufd/vdevice: Add TSM map ioctl

In reply to Alexey Kardashevskiy
Quoted 16 lines · Alexey Kardashevskiy
Quoted 7 lines · Jason Gunthorpe
Quoted 5 lines · Aneesh Kumar K.V (Arm)

With passthrough devices, we need to make sure private memory is allocated and assigned to the secure guest before we can issue the DMA. For ARM RMM, we only need to map and the secure SMMU management is internal to RMM. For shared IPA, vfio/iommufd DMA MAP/UNMAP interface does the equivalent

Quoted 6 lines

I'm not really sure what this is about? It is about getting KVM to pin all the memory and commit it to the RMM so it can be used for DMA?

But it looks really strange to have an iommufd ioctl that just calls a KVM function. Feeling this should be a KVM function, or a guestmfd behavior??

Quoted 5 lines

I ended up exporting the guestmemfd's kvm_gmem_get_folio() for gfn->pfn and its fd a bit differently in iommufd - "no extra referencing":
https://github.com/AMDESE/linux-kvm/commit/f1ebd358327f026f413f8d3d64d46decfd6ab7f6

It is a new iommufd->kvm dependency though.

Was the motivation for that design choice the fact that in case of AMD VFIO/IOMMUFD manages both private memory allocation and updates to the IOMMU page tables?

On the ARM side, the requirement is to ensure that pages are present in the stage-2 page table, which is managed by the firmware (RMM). Because of this, we need an interface that VFIO/IOMMUFD can use to trigger stage-2 mappings within KVM.

Alternatively, we could introduce a dedicated KVM ioctl for this purpose, avoiding the need to rely on IOMMUFD.

For reference, TDX uses a similar ioctl—KVM_TDX_INIT_MEM_REGION—to initialize guest memory. However, that interface isn’t well-suited for dynamic updates to stage-2 mappings during shared-to-private or private-to-shared transitions.

Quoted 6 lines
Quoted 5 lines

I was kind of thinking it would be nice to have a guestmemfd mode that was "pinned", meaning the memory is allocated and remains almost always mapped into the TSM's page tables automatically. VFIO using guests would set things this way.

Quoted 7 lines

Yeah while doing the above, I was wondering if I want to pass the fd type when DMA-mapping from an fd or "detect" it as I do in the above commit or have some iommufd_fdmap_ops in this fd saying "(no) pinning needed" (or make this a flag of IOMMU_IOAS_MAP_FILE).

The "detection" is (mapping_inaccessible(mapping) && mapping_unevictable(mapping)), works for now.

btw in the AMD case, here it does not matter as much if it is private or shared, I map everything and let RMP and the VM deal with the permissions. Thanks,

-aneesh

#172 lore raw

Re: [RFC PATCH v1 38/38] coco: guest: arm64: Add support for fetching device info

In reply to Aneesh Kumar K.V
Quoted 26 lines
Quoted 16 lines
Quoted 16 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +14 -0
+	if (!dev_info) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	ret = rsi_rdev_get_info(vdev_id, dsm->instance_id, virt_to_phys(dev_info));
+	if (ret != RSI_SUCCESS) {
+		pci_err(pdev, "failed to get device digests (%lu)\n", ret);
+		ret = -EIO;
+		kfree(dev_info);
+		goto err_out;
+	}
+
+	dsm->dev_info.attest_type   = dev_info->attest_type;
Quoted 9 lines

drivers/virt/coco/arm-cca-guest/rsi-da.c int rsi_device_lock(struct pci_dev *pdev) · line 368

Quoted 6 lines
drivers/virt/coco/arm-cca-guest/rsi-da.c +6 -0
+	dsm->dev_info.cert_id       = dev_info->cert_id;
+	dsm->dev_info.hash_algo     = dev_info->hash_algo;
+	memcpy(dsm->dev_info.cert_digest, dev_info->cert_digest, SHA512_DIGEST_SIZE);
+	memcpy(dsm->dev_info.meas_digest, dev_info->meas_digest, SHA512_DIGEST_SIZE);
+	memcpy(dsm->dev_info.report_digest, dev_info->report_digest, SHA512_DIGEST_SIZE);
+  

Can't you memcpy the whole thing in one go?

Quoted 3 lines

yes. But won't that be confusing? Is there a difference? Also struct dsm_device_info is not same as struct rsi_device_info. We don't need to keep all that padding in dsm_device_info.

Ah. I misread and thought they were the same structure. No problem copying only relevant fields then!

Jonathan

#4 lore raw

Re: [RFC PATCH v1 03/38] tsm: Move dsm_dev from pci_tdi to pci_tsm

In reply to Aneesh Kumar K.V (Arm)

On Mon, Jul 28, 2025 at 07:21:40PM +0530, Aneesh Kumar K.V (Arm) wrote:

Subject line should include "PCI" prefix.

Needs a commit log, even if it repeats what's in the subject. Would also be good to know *why* this is desirable.

Quoted 11 lines
---
 drivers/pci/tsm.c       | 72 ++++++++++++++++++++++++-----------------
 include/linux/pci-tsm.h |  4 +--
 2 files changed, 45 insertions(+), 31 deletions(-)
drivers/pci/tsm.c +0 -0
index 794de2f258c3..e4a3b5b37939 100644

drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 418

Quoted 4 lines
drivers/pci/tsm.c +1 -0
@@ -415,15 +415,55 @@ static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev)
 	return PCI_TSM_INVALID;
 }
 
+/* lookup the Device Security Manager (DSM) pf0 for @pdev */

s/lookup/Look up/ (it's a verb here)

drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 422

Quoted 4 lines
drivers/pci/tsm.c +4 -0
+static struct pci_dev *dsm_dev_get(struct pci_dev *pdev)
+{
+	struct pci_dev *uport_pf0;
+

Remove this blank line ...

drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 423

Quoted 1 line
drivers/pci/tsm.c +1 -0
+	struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev);

and add one here.

Quoted 4 lines
drivers/pci/tsm.c +4 -0
+	if (!pf0)
+		return NULL;
+
+	if (pf0 == pdev)

drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 437

Quoted 10 lines
drivers/pci/tsm.c +10 -0
+		return no_free_ptr(pf0);
+
+	/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */
+	if (pf0->tsm && pf0->tsm->type == PCI_TSM_PF0)
+		return no_free_ptr(pf0);
+
+	/*
+	 * For cases where a switch may be hosting TDISP services on
+	 * behalf of downstream devices, check the first usptream port
+	 * relative to this endpoint.

s/usptream/upstream/

Quoted 14 lines
drivers/pci/tsm.c +10 -0
+	 */
+	if (!pdev->dev.parent || !pdev->dev.parent->parent)
+		return NULL;
+
+	uport_pf0 = to_pci_dev(pdev->dev.parent->parent);
+	if (!uport_pf0->tsm)
+		return NULL;
+	return pci_dev_get(uport_pf0);
+}
+
 /**
  * pci_tsm_initialize() - base 'struct pci_tsm' initialization
  * @pdev: The PCI device
  * @tsm: context to initialize

drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 460

Quoted 10 lines
drivers/pci/tsm.c +5 -1
  */
-void pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm)
+int pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm)
 {
+	struct pci_dev *dsm_dev __free(pci_dev_put) = dsm_dev_get(pdev);
+	if (!dsm_dev)
+		return -EINVAL;
+
 	tsm->type = pci_tsm_type(pdev);
 	tsm->pdev = pdev;

Add blank line before comment.

drivers/pci/tsm.c static enum pci_tsm_type pci_tsm_type(struct pci_dev *pdev) · line 463

Quoted 3 lines
drivers/pci/tsm.c +3 -0
+	/*
+	 * No reference needed because when we destroy
+	 * dsm_dev all the tdis get destroyed before that.

"tdi" looks like an initialism, which would normally be capitalized.

Quoted 36 lines
drivers/pci/tsm.c +5 -17
+	 */
+	tsm->dsm_dev = dsm_dev;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pci_tsm_initialize);
 
@@ -447,7 +487,8 @@ int pci_tsm_pf0_initialize(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm)
 	}
 
 	tsm->state = PCI_TSM_INIT;
-	pci_tsm_initialize(pdev, &tsm->tsm);
+	if (pci_tsm_initialize(pdev, &tsm->tsm))
+		return -ENODEV;
 
 	return 0;
 }
@@ -612,32 +653,6 @@ int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type,
 }
 EXPORT_SYMBOL_GPL(pci_tsm_doe_transfer);
 
-/* lookup the Device Security Manager (DSM) pf0 for @pdev */
-static struct pci_dev *dsm_dev_get(struct pci_dev *pdev)
-{
-	struct pci_dev *uport_pf0;
-
-	struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev);
-	if (!pf0)
-		return NULL;
-
-	/* Check that @pf0 was not initialized as PCI_TSM_DOWNSTREAM */
-	if (pf0->tsm && pf0->tsm->type == PCI_TSM_PF0)
-		return no_free_ptr(pf0);
-
-	/*
-	 * For cases where a switch may be hosting TDISP services on
-	 * behalf of downstream devices, check the first usptream port

drivers/pci/tsm.c int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type, · line 640

Quoted 10 lines
drivers/pci/tsm.c +0 -10
-	 * relative to this endpoint.
-	 */
-	if (!pdev->dev.parent || !pdev->dev.parent->parent)
-		return NULL;
-
-	uport_pf0 = to_pci_dev(pdev->dev.parent->parent);
-	if (!uport_pf0->tsm)
-		return NULL;
-	return pci_dev_get(uport_pf0);
-}

This code move looks like it could be a separate patch that only moves (and fixes the typos I mentioned).

Then a second patch could do what the subject claims (moving dsm_dev from pci_tdi to pci_tsm) so it's not buried in the simple move.

Quoted 42 lines
drivers/pci/tsm.c +0 -1
 /* Only implement non-interruptible lock for now */
 static struct mutex *tdi_ops_lock(struct pci_dev *pf0_dev)
@@ -695,7 +710,6 @@ int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id)
 		return -ENXIO;
 
 	tdi->pdev = pdev;
-	tdi->dsm_dev = dsm_dev;
 	tdi->kvm = kvm;
 	pdev->tsm->tdi = tdi;
 
include/linux/pci-tsm.h +2 -2
index 1920ca591a42..0d4303726b25 100644
@@ -38,7 +38,6 @@ enum pci_tsm_type {
  */
 struct pci_tdi {
 	struct pci_dev *pdev;
-	struct pci_dev *dsm_dev;
 	struct kvm *kvm;
 };
 
@@ -56,6 +55,7 @@ struct pci_tdi {
  */
 struct pci_tsm {
 	struct pci_dev *pdev;
+	struct pci_dev *dsm_dev;
 	enum pci_tsm_type type;
 	struct pci_tdi *tdi;
 };
@@ -173,7 +173,7 @@ void pci_tsm_core_unregister(const struct pci_tsm_ops *ops);
 int pci_tsm_doe_transfer(struct pci_dev *pdev, enum pci_doe_proto type,
 			 const void *req, size_t req_sz, void *resp,
 			 size_t resp_sz);
-void pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm);
+int pci_tsm_initialize(struct pci_dev *pdev, struct pci_tsm *tsm);
 int pci_tsm_pf0_initialize(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm);
 int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id);
 int pci_tsm_unbind(struct pci_dev *pdev);
Signature · 1 line
2.43.0
#22 lore raw

Re: [RFC PATCH v1 04/38] tsm: Support DMA Allocation from private memory

In reply to Aneesh Kumar K.V (Arm)

drivers/pci/tsm.c static void __pci_tsm_init(struct pci_dev *pdev) · line 564

Quoted 10 lines · Aneesh Kumar K.V (Arm)

Currently, we enforce the use of bounce buffers to ensure that memory accessed by non-secure devices is explicitly shared with the host [1]. However, for secure devices, this approach must be avoided.

drivers/pci/tsm.c +2 -0
@@ -557,6 +560,9 @@ static void __pci_tsm_init(struct pci_dev *pdev)
 	default:
 		break;
 	}
+
+	/* FIXME!! should this be default true and switch to false for TEE capable device */

Fix whatever this is, or make it a real sentence and wrap to fit in 80 columns.

Quoted 2 lines
(quoted diff) +1 -0
+	pdev->dev.tdi_enabled = false;
 }
#27 lore raw

Re: [RFC PATCH v1 05/38] tsm: Don't overload connect

In reply to Aneesh Kumar K.V (Arm)

Only touches drivers/pci, so needs a "PCI" prefix of some flavor.

Quoted 5 lines · Aneesh Kumar K.V (Arm)

We need separate handling in the guest while destroying the device. Hence switch to new callback lock and unlock. Hide the connect sysfs file in guest.

guest sysfs will now looks like

s/guest/Guest/
s/will now looks/will now look/

Review

Quoted 6 lines

ls -al /sys/bus/pci/devices/0000:02:00.0/tsm/
total 0

drwxr-xr-x    2 root     root             0 Jan  1 00:00 .
drwxr-xr-x    7 root     root             0 Jan  1 00:00 ..
(quoted diff) +0 -2
-rw-r--r--    1 root     root          4096 Jan  1 00:00 accept
-rw-r--r--    1 root     root          4096 Jan  1 00:00 lock

Indent quoted material a couple spaces.

I don't know about TSM, so this commit log doesn't quite tell me why we need this.

It appears this also adds a sysfs file that could be documented somehow.

Quoted 42 lines
---
 drivers/pci/tsm.c       | 136 ++++++++++++++++++++++++++++++++++------
 include/linux/pci-tsm.h |   3 +
 2 files changed, 121 insertions(+), 18 deletions(-)
drivers/pci/tsm.c +1 -16
index 60f50d57a725..80607082b7f0 100644
@@ -125,18 +125,6 @@ static int pci_tsm_disconnect(struct pci_dev *pdev)
 	return 0;
 }
 
-/*
- * TDISP locked state temporarily makes the device inaccessible, do not
- * surprise live attached drivers
- */
-static int __driver_idle_connect(struct pci_dev *pdev)
-{
-	guard(device)(&pdev->dev);
-	if (pdev->dev.driver)
-		return -EBUSY;
-	return tsm_ops->connect(pdev);
-}
-
 /*
  * When the registered ops support accept it indicates that this is a
  * TVM-side (guest) TSM operations structure. In this mode ->connect()
@@ -162,10 +150,7 @@ static int pci_tsm_connect(struct pci_dev *pdev)
 	if (tsm->state >= PCI_TSM_CONNECT)
 		return 0;
 
-	if (tvm_mode())
-		rc = __driver_idle_connect(pdev);
-	else
-		rc = tsm_ops->connect(pdev);
+	rc = tsm_ops->connect(pdev);
 	if (rc)
 		return rc;
 	tsm->state = PCI_TSM_CONNECT;

drivers/pci/tsm.c static ssize_t accept_show(struct device *dev, struct device_attribute *attr, · line 290

Quoted 7 lines
drivers/pci/tsm.c +4 -0
@@ -299,6 +284,99 @@ static ssize_t accept_show(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR_RW(accept);
 
+static int pci_tsm_unlock(struct pci_dev *pdev)
+{
+	struct pci_tsm_pf0 *tsm = to_pci_tsm_pf0(pdev->tsm);
+

Drop this blank line ...

drivers/pci/tsm.c static ssize_t accept_show(struct device *dev, struct device_attribute *attr, · line 291

Quoted 1 line
drivers/pci/tsm.c +1 -0
+	struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm);

add one here. I'm seeing a new pattern; maybe the trend is to put declarations of __free() things outside the usual local variable list?

Quoted 176 lines
drivers/pci/tsm.c +112 -2
+	if (!lock)
+		return -EINTR;
+
+	if (tsm->state < PCI_TSM_INIT)
+		return -ENXIO;
+	if (tsm->state < PCI_TSM_LOCK)
+		return 0;
+
+	tsm_ops->unlock(pdev);
+	tsm->state = PCI_TSM_INIT;
+	pdev->dev.tdi_enabled = false;
+
+	return 0;
+}
+
+/*
+ * TDISP locked state temporarily makes the device inaccessible, do not
+ * surprise live attached drivers
+ */
+static int __driver_idle_lock(struct pci_dev *pdev)
+{
+	guard(device)(&pdev->dev);
+	if (pdev->dev.driver)
+		return -EBUSY;
+	return tsm_ops->lock(pdev);
+}
+
+static int pci_tsm_lock(struct pci_dev *pdev)
+{
+	struct pci_tsm_pf0 *tsm = to_pci_tsm_pf0(pdev->tsm);
+	int rc;
+
+	struct mutex *lock __free(tsm_ops_unlock) = tsm_ops_lock(tsm);
+	if (!lock)
+		return -EINTR;
+
+	if (tsm->state < PCI_TSM_INIT)
+		return -ENXIO;
+
+	rc = __driver_idle_lock(pdev);
+	if (rc)
+		return rc;
+	tsm->state = PCI_TSM_CONNECT;
+	return 0;
+}
+
+static ssize_t lock_store(struct device *dev, struct device_attribute *attr,
+			  const char *buf, size_t len)
+{
+	int rc;
+	bool connect;
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	rc = kstrtobool(buf, &connect);
+	if (rc)
+		return rc;
+
+	struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock();
+	if (!lock)
+		return -EINTR;
+
+	if (connect)
+		rc = pci_tsm_lock(pdev);
+	else
+		rc = pci_tsm_unlock(pdev);
+	if (rc)
+		return rc;
+	return len;
+}
+
+static ssize_t lock_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pci_tsm_pf0 *tsm;
+
+	struct rw_semaphore *lock __free(tsm_read_unlock) = tsm_read_lock();
+	if (!lock)
+		return -EINTR;
+
+	if (!pdev->tsm)
+		return -ENXIO;
+
+	tsm = to_pci_tsm_pf0(pdev->tsm);
+	return sysfs_emit(buf, "%d\n", tsm->state >= PCI_TSM_LOCK);
+}
+static DEVICE_ATTR_RW(lock);
+
 static umode_t pci_tsm_pf0_attr_visible(struct kobject *kobj,
 					struct attribute *a, int n)
 {
@@ -306,6 +384,11 @@ static umode_t pci_tsm_pf0_attr_visible(struct kobject *kobj,
 		/* Host context, filter out guest only attributes */
 		if (a == &dev_attr_accept.attr)
 			return 0;
+		if (a == &dev_attr_lock.attr)
+			return 0;
+	} else {
+		if (a == &dev_attr_connect.attr)
+			return 0;
 	}
 
 	return a->mode;
@@ -325,6 +408,7 @@ DEFINE_SYSFS_GROUP_VISIBLE(pci_tsm_pf0);
 static struct attribute *pci_tsm_pf0_attrs[] = {
 	&dev_attr_connect.attr,
 	&dev_attr_accept.attr,
+	&dev_attr_lock.attr,
 	NULL
 };
 
@@ -537,7 +621,8 @@ static void pci_tsm_pf0_init(struct pci_dev *pdev)
 		return;
 
 	pdev->tsm = no_free_ptr(pci_tsm);
-	sysfs_update_group(&pdev->dev.kobj, &pci_tsm_auth_attr_group);
+	if (!tvm_mode())
+		sysfs_update_group(&pdev->dev.kobj, &pci_tsm_auth_attr_group);
 	sysfs_update_group(&pdev->dev.kobj, &pci_tsm_pf0_attr_group);
 	if (pci_tsm_owner_attr_group)
 		sysfs_merge_group(&pdev->dev.kobj, pci_tsm_owner_attr_group);
@@ -602,6 +687,19 @@ static void pci_tsm_pf0_destroy(struct pci_dev *pdev)
 	__pci_tsm_pf0_destroy(tsm);
 }
 
+static void pci_tsm_guest_destroy(struct pci_dev *pdev)
+{
+	struct pci_tsm_pf0 *tsm = to_pci_tsm_pf0(pdev->tsm);
+
+	if (tsm->state > PCI_TSM_INIT)
+		pci_tsm_unlock(pdev);
+	pdev->tsm = NULL;
+	if (pci_tsm_owner_attr_group)
+		sysfs_unmerge_group(&pdev->dev.kobj, pci_tsm_owner_attr_group);
+	sysfs_update_group(&pdev->dev.kobj, &pci_tsm_pf0_attr_group);
+	__pci_tsm_pf0_destroy(tsm);
+}
+
 static void __pci_tsm_destroy(struct pci_dev *pdev)
 {
 	struct pci_tsm *pci_tsm = pdev->tsm;
@@ -611,7 +709,9 @@ static void __pci_tsm_destroy(struct pci_dev *pdev)
 
 	lockdep_assert_held_write(&pci_tsm_rwsem);
 
-	if (is_pci_tsm_pf0(pdev)) {
+	if (tvm_mode()) {
+		pci_tsm_guest_destroy(pdev);
+	} else if (is_pci_tsm_pf0(pdev)) {
 		pci_tsm_pf0_destroy(pdev);
 	} else {
 		__pci_tsm_unbind(pdev);
include/linux/pci-tsm.h +3 -0
index 0d4303726b25..7639e7963681 100644
@@ -11,6 +11,7 @@ enum pci_tsm_state {
 	PCI_TSM_ERR = -1,
 	PCI_TSM_INIT,
 	PCI_TSM_CONNECT,
+	PCI_TSM_LOCK,
 	PCI_TSM_ACCEPT,
 };
 
@@ -153,6 +154,8 @@ struct pci_tsm_ops {
 	void (*remove)(struct pci_tsm *tsm);
 	int (*connect)(struct pci_dev *pdev);
 	void (*disconnect)(struct pci_dev *pdev);
+	int (*lock)(struct pci_dev *pdev);
+	void (*unlock)(struct pci_dev *pdev);
 	struct pci_tdi *(*bind)(struct pci_dev *pdev, struct pci_dev *pf0_dev,
 				struct kvm *kvm, u64 tdi_id);
 	void (*unbind)(struct pci_tdi *tdi);
Signature · 1 line
2.43.0