intel_iommu: Add ATS support
v5 cover intel_iommu: Add ATS support
This patch set belongs to a list of series that add SVM support for VT-d. Here we focus on implementing the ATS API exposed through the PCIIOMMUOps callbacks.
This work is based on the VT-d specification version 4.1 (March 2023).
Here is a link to our GitHub repository where you can find:
- Qemu with all the patches for SVM
- ATS
- PRI
- Device IOTLB invalidations
- Requests with already pre-translated addresses
- A demo device
- A simple driver for the demo device
- A userspace program (for testing and demonstration purposes)
https://github.com/BullSequana/Qemu-in-guest-SVM-demo
What is ATS?
''''''''''''
ATS (Address Translation Service) is a PCIe-level protocol that enables PCIe devices to query an IOMMU for virtual to physical address translations in a specific address space (potentially identified by a PASID). When a device receives a translation response from an IOMMU, it may decide to store it in an internal cache, often known as "ATC" (Address Translation Cache) or "Device IOTLB". When triggering a memory operation using a physical address obtained via ATS, the device must set the AT field of the PCIe message to "translated" to prevent the IOMMU from translating the address again. To keep page tables and caches consistent, the host system can ask the IOMMU to send invalidation requests to the devices.
Changelog:
v5
- Migrate to the PCIIOMMUOps API
v4
- Fix compilation issue on 32bit hosts
v3
- Fix compilation issue in non-debug mode (Michael)
v2
- Rebase on master after merge of Zhenzhong's FLTS series
- Rename the series as it is now based on master.
- Changes after review by Michael:
- Split long lines in memory.h
- Change patch encoding (no UTF-8)
- Changes after review by Zhenzhong:
- Rework "Fill the PASID field when creating an IOMMUTLBEntry"
Clement Mathieu--Drif (10):
pci: Add a memory attribute for pre-translated DMA operations
memory: Add permissions in IOMMUAccessFlags
memory: Allow to store the PASID in IOMMUTLBEntry
intel_iommu: Fill the PASID field when creating an IOMMUTLBEntry
intel_iommu: Declare supported PASID size
intel_iommu: Implement vtd_get_iotlb_info from PCIIOMMUOps
intel_iommu: Implement the PCIIOMMUOps callbacks related to
invalidations of device-IOTLB
intel_iommu: Return page walk level even when the translation fails
intel_iommu: Set address mask when a translation fails and adjust W
permission
intel_iommu: Add support for ATS
hw/i386/intel_iommu.c | 142 ++++++++++++++++++++++++++++++--- hw/i386/intel_iommu_internal.h | 1 + include/exec/memattrs.h | 3 + include/hw/pci/pci.h | 9 +++ include/system/memory.h | 24 +++++- 5 files changed, 164 insertions(+), 15 deletions(-)
Signature · 1 line
2.49.0
PATCH v5 1/10 pci: Add a memory attribute for pre-translated DMA operations
The address_type bit will be set to PCI_AT_TRANSLATED by devices that use cached addresses obtained via ATS.
--- include/exec/memattrs.h | 3 +++ include/hw/pci/pci.h | 9 +++++++++ 2 files changed, 12 insertions(+)
include/exec/memattrs.h +3 -0
@@ -54,6 +54,9 @@ typedef struct MemTxAttrs {*/ unsigned int pid:8; +/* PCI - IOMMU operations, see PCIAddressType */ + unsigned int address_type:1; +/* * Bus masters which don't specify any attributes will get this * (via the MEMTXATTRS_UNSPECIFIED constant), so that we can
include/hw/pci/pci.h +9 -0
@@ -134,6 +134,15 @@ struct PCIHostDeviceAddress {unsigned int function; }; +/* +* Represents the Address Type (AT) field in a PCI request, +* see MemTxAttrs.address_type +*/ +typedef enum PCIAddressType { +PCI_AT_UNTRANSLATED =0 ,/* Default when no attribute is set */ +PCI_AT_TRANSLATED =1 , +}PCIAddressType ; +typedef void PCIConfigWriteFunc (PCIDevice *pci_dev,uint32_t address,uint32_t data,int len); typedef uint32_t PCIConfigReadFunc(PCIDevice *pci_dev,
Signature · 1 line
2.49.0
PATCH v5 2/10 memory: Add permissions in IOMMUAccessFlags
In reply to CLEMENT MATHIEU--DRIFThis will be necessary for devices implementing ATS.
We also define a new macro IOMMU_ACCESS_FLAG_FULL in addition to IOMMU_ACCESS_FLAG to support more access flags. IOMMU_ACCESS_FLAG is kept for convenience and backward compatibility.
Here are the flags added (defined by the PCIe 5 specification) :
- Execute Requested
- Privileged Mode Requested
- Global
- Untranslated Only
IOMMU_ACCESS_FLAG sets the additional flags to 0
--- include/system/memory.h | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-)
include/system/memory.h +21 -2
@@ -109,15 +109,34 @@ struct MemoryRegionSection {typedef struct IOMMUTLBEntry IOMMUTLBEntry ; -/* See address_space_translate: bit 0 is read, bit 1 is write. */ +/* +* See address_space_translate: +* - bit 0 : read +* - bit 1 : write +* - bit 2 : exec +* - bit 3 : priv +* - bit 4 : global +* - bit 5 : untranslated only +*/ typedef enum {IOMMU_NONE =0 ,IOMMU_RO =1 ,IOMMU_WO =2 ,IOMMU_RW =3 , +IOMMU_EXEC =4 , +IOMMU_PRIV =8 , +IOMMU_GLOBAL =16 , +IOMMU_UNTRANSLATED_ONLY =32 , }IOMMUAccessFlags ; -#define IOMMU_ACCESS_FLAG (r, w) (((r) ? IOMMU_RO : 0) | ((w) ? IOMMU_WO : 0)) +#define IOMMU_ACCESS_FLAG (r, w) (((r) ? IOMMU_RO : 0) | \ + ((w) ? IOMMU_WO : 0)) +#define IOMMU_ACCESS_FLAG_FULL (r, w, x, p, g, uo) \ + (IOMMU_ACCESS_FLAG(r, w) | \ + ((x) ? IOMMU_EXEC : 0) | \ + ((p) ? IOMMU_PRIV : 0) | \ + ((g) ? IOMMU_GLOBAL : 0) | \ + ((uo) ? IOMMU_UNTRANSLATED_ONLY : 0))struct IOMMUTLBEntry {AddressSpace *target_as;
Signature · 1 line
2.49.0
PATCH v5 3/10 memory: Allow to store the PASID in IOMMUTLBEntry
In reply to CLEMENT MATHIEU--DRIFThis will be useful for devices that support ATS and need to store entries in an ATC (device IOTLB).
--- include/system/memory.h | 1 + 1 file changed, 1 insertion(+)
include/system/memory.h +1 -0
@@ -144,6 +144,7 @@ struct IOMMUTLBEntry {hwaddr translated_addr;hwaddr addr_mask;/* 0xfff = 4k translation */ IOMMUAccessFlags perm; +uint32_t pasid; };/*
Signature · 1 line
2.49.0
PATCH v5 4/10 intel_iommu: Fill the PASID field when creating an IOMMUTLBEntry
In reply to CLEMENT MATHIEU--DRIFPASID value must be used by devices as a key (or part of a key) when populating their ATC with the IOTLB entries returned by the IOMMU.
--- hw/i386/intel_iommu.c | 3 +++ 1 file changed, 3 insertions(+)
hw/i386/intel_iommu.c +3 -0
@@ -2503,6 +2503,7 @@ static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, .translated_addr = 0, .addr_mask = size - 1, .perm = IOMMU_NONE, + .pasid = vtd_as->pasid, }, };memory_region_notify_iommu (&vtd_as->iommu,0 , event);
@@ -3090,6 +3091,7 @@ static void do_invalidate_device_tlb(VTDAddressSpace *vtd_dev_as, event.entry.iova = addr; event.entry.perm =IOMMU_NONE ; event.entry.translated_addr =0 ; + event.entry.pasid = vtd_dev_as->pasid;memory_region_notify_iommu (&vtd_dev_as->iommu,0 , event); }
@@ -3672,6 +3674,7 @@ static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr,IOMMUTLBEntry iotlb = {/* We'll fill in the rest later. */ .target_as = &address_space_memory, + .pasid = vtd_as->pasid, };bool success;
Signature · 1 line
2.49.0
PATCH v5 5/10 intel_iommu: Declare supported PASID size
In reply to CLEMENT MATHIEU--DRIFthe PSS field of the extended capabilities stores the supported PASID size minus 1. This commit adds support for 8bits PASIDs (limited by MemTxAttrs::pid).
--- hw/i386/intel_iommu.c | 2 +- hw/i386/intel_iommu_internal.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-)
hw/i386/intel_iommu.c +1 -1
@@ -4590,7 +4590,7 @@ static void vtd_cap_init(IntelIOMMUState *s) }if (s->pasid) { - s->ecap |=VTD_ECAP_PASID ; + s->ecap |=VTD_ECAP_PASID |VTD_ECAP_PSS ; } }
hw/i386/intel_iommu_internal.h +1 -0
@@ -192,6 +192,7 @@#define VTD_ECAP_SC (1ULL << 7)#define VTD_ECAP_MHMV (15ULL << 20)#define VTD_ECAP_SRS (1ULL << 31) +#define VTD_ECAP_PSS (7ULL << 35)/* limit: MemTxAttrs::pid */ #define VTD_ECAP_PASID (1ULL << 40)#define VTD_ECAP_SMTS (1ULL << 43)#define VTD_ECAP_SLTS (1ULL << 46)
Signature · 1 line
2.49.0
PATCH v5 6/10 intel_iommu: Implement vtd_get_iotlb_info from PCIIOMMUOps
In reply to CLEMENT MATHIEU--DRIF--- hw/i386/intel_iommu.c | 10 ++++++++++ 1 file changed, 10 insertions(+)
hw/i386/intel_iommu.c +10 -0
@@ -4733,10 +4733,20 @@ static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)return &vtd_as->as; } +static void vtd_get_iotlb_info (void *opaque,uint8_t *addr_width, +uint32_t *min_page_size) +{ +IntelIOMMUState *s = opaque; + + *addr_width = s->aw_bits; + *min_page_size =VTD_PAGE_SIZE ; +} +static PCIIOMMUOps vtd_iommu_ops = { .get_address_space = vtd_host_dma_iommu, .set_iommu_device = vtd_dev_set_iommu_device, .unset_iommu_device = vtd_dev_unset_iommu_device, + .get_iotlb_info = vtd_get_iotlb_info, };static bool vtd_decide_config (IntelIOMMUState *s,Error **errp)
Signature · 1 line
2.49.0
PATCH v5 7/10 intel_iommu: Implement the PCIIOMMUOps callbacks related to invalidations of device-IOTLB
In reply to CLEMENT MATHIEU--DRIF--- hw/i386/intel_iommu.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+)
hw/i386/intel_iommu.c +35 -0
@@ -4733,6 +4733,15 @@ static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)return &vtd_as->as; } +static void vtd_init_iotlb_notifier (PCIBus *bus,void *opaque,int devfn, +IOMMUNotifier *n,IOMMUNotify fn, +void *user_opaque) +{ + n->opaque = user_opaque; +iommu_notifier_init (n, fn,IOMMU_NOTIFIER_DEVIOTLB_EVENTS ,0 , +HWADDR_MAX ,0 ); +} +static void vtd_get_iotlb_info (void *opaque,uint8_t *addr_width,uint32_t *min_page_size) {
@@ -4742,11 +4751,37 @@ static void vtd_get_iotlb_info(void *opaque, uint8_t *addr_width, *min_page_size =VTD_PAGE_SIZE ; } +static void vtd_register_iotlb_notifier (PCIBus *bus,void *opaque, +int devfn,uint32_t pasid, +IOMMUNotifier *n) +{ +IntelIOMMUState *s = opaque; +VTDAddressSpace *vtd_as; + + vtd_as =vtd_find_add_as (s, bus, devfn, pasid); +memory_region_register_iommu_notifier (MEMORY_REGION (&vtd_as->iommu), n, + &error_fatal); +} + +static void vtd_unregister_iotlb_notifier (PCIBus *bus,void *opaque, +int devfn,uint32_t pasid, +IOMMUNotifier *n) +{ +IntelIOMMUState *s = opaque; +VTDAddressSpace *vtd_as; + + vtd_as =vtd_find_add_as (s, bus, devfn, pasid); +memory_region_unregister_iommu_notifier (MEMORY_REGION (&vtd_as->iommu), n); +} +static PCIIOMMUOps vtd_iommu_ops = { .get_address_space = vtd_host_dma_iommu, .set_iommu_device = vtd_dev_set_iommu_device, .unset_iommu_device = vtd_dev_unset_iommu_device, .get_iotlb_info = vtd_get_iotlb_info, + .init_iotlb_notifier = vtd_init_iotlb_notifier, + .register_iotlb_notifier = vtd_register_iotlb_notifier, + .unregister_iotlb_notifier = vtd_unregister_iotlb_notifier, };static bool vtd_decide_config (IntelIOMMUState *s,Error **errp)
Signature · 1 line
2.49.0
PATCH v5 8/10 intel_iommu: Return page walk level even when the translation fails
In reply to CLEMENT MATHIEU--DRIFWe will use this information in vtd_do_iommu_translate to populate the IOMMUTLBEntry and indicate the correct page mask. This prevents ATS devices from sending many useless translation requests when a megapage or gigapage is not present.
--- hw/i386/intel_iommu.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
hw/i386/intel_iommu.c +8 -9
@@ -1987,9 +1987,9 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce, uint32_t pasid) {dma_addr_t addr =vtd_get_iova_pgtbl_base (s, ce, pasid); -uint32_t level =vtd_get_iova_level (s, ce, pasid);uint32_t offset;uint64_t flpte, flag_ad =VTD_FL_A ; + *flpte_level =vtd_get_iova_level (s, ce, pasid);if (!vtd_iova_fl_check_canonical (s, iova, ce, pasid)) { error_report_once("%s: detected non canonical IOVA (iova=0x%" PRIx64 ","
@@ -1998,11 +1998,11 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce, }while (true) { - offset =vtd_iova_level_offset (iova, level); + offset =vtd_iova_level_offset (iova, *flpte_level); flpte =vtd_get_pte (addr, offset);if (flpte == (uint64_t )-1 ) { -if (level ==vtd_get_iova_level (s, ce, pasid)) { +if (*flpte_level ==vtd_get_iova_level (s, ce, pasid)) {/* Invalid programming of pasid-entry */ return -VTD_FR_PASID_ENTRY_FSPTPTR_INV ; }else {
@@ -2028,15 +2028,15 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce,if (is_write && !(flpte &VTD_FL_RW )) {return -VTD_FR_SM_WRITE ; } -if (vtd_flpte_nonzero_rsvd (flpte, level)) { +if (vtd_flpte_nonzero_rsvd (flpte, *flpte_level)) {error_report_once ("%s: detected flpte reserved non-zero " "iova=0x%" PRIx64", level=0x%" PRIx32"flpte=0x%" PRIx64", pasid=0x%" PRIX32 ")" , - __func__, iova, level, flpte, pasid); + __func__, iova, *flpte_level, flpte, pasid);return -VTD_FR_FS_PAGING_ENTRY_RSVD ; } -if (vtd_is_last_pte (flpte, level) && is_write) { +if (vtd_is_last_pte (flpte, *flpte_level) && is_write) { flag_ad |=VTD_FL_D ; }
@@ -2044,14 +2044,13 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce,return -VTD_FR_FS_BIT_UPDATE_FAILED ; } -if (vtd_is_last_pte (flpte, level)) { +if (vtd_is_last_pte (flpte, *flpte_level)) { *flptep = flpte; - *flpte_level = level;return 0 ; } addr =vtd_get_pte_addr (flpte, aw_bits); - level--; + (*flpte_level)--; } }
Signature · 1 line
2.49.0
PATCH v5 9/10 intel_iommu: Set address mask when a translation fails and adjust W permission
In reply to CLEMENT MATHIEU--DRIFImplements the behavior defined in section 10.2.3.5 of PCIe spec rev 5. This is needed by devices that support ATS.
--- hw/i386/intel_iommu.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-)
hw/i386/intel_iommu.c +9 -3
@@ -2091,7 +2091,8 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,uint8_t bus_num =pci_bus_num (bus);VTDContextCacheEntry *cc_entry;uint64_t pte, page_mask; -uint32_t level, pasid = vtd_as->pasid; +uint32_t level =UINT32_MAX ; +uint32_t pasid = vtd_as->pasid;uint16_t source_id =PCI_BUILD_BDF (bus_num, devfn);int ret_fr;bool is_fpd_set = false;
@@ -2250,14 +2251,19 @@ out: entry->iova = addr & page_mask; entry->translated_addr =vtd_get_pte_addr (pte, s->aw_bits) & page_mask; entry->addr_mask = ~page_mask; - entry->perm = access_flags; + entry->perm = (is_write ? access_flags : (access_flags & (~IOMMU_WO )));return true; error:vtd_iommu_unlock (s); entry->iova =0 ; entry->translated_addr =0 ; - entry->addr_mask =0 ; +/* +* Set the mask for ATS (the range must be present even when the +* translation fails : PCIe rev 5 10.2.3.5) +*/ + entry->addr_mask = (level !=UINT32_MAX ) ? + (~vtd_pt_level_page_mask (level)) : (~VTD_PAGE_MASK_4K ); entry->perm =IOMMU_NONE ;return false; }
Signature · 1 line
2.49.0
PATCH v5 10/10 intel_iommu: Add support for ATS
In reply to CLEMENT MATHIEU--DRIF--- hw/i386/intel_iommu.c | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+)
hw/i386/intel_iommu.c +63 -0
@@ -4738,6 +4738,68 @@ static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)return &vtd_as->as; } +static IOMMUTLBEntry vtd_iommu_ats_do_translate (IOMMUMemoryRegion *iommu, +hwaddr addr, +IOMMUAccessFlags flags) +{ +IOMMUTLBEntry entry; +VTDAddressSpace *vtd_as =container_of (iommu, VTDAddressSpace, iommu); + +if (vtd_is_interrupt_addr (addr)) { +vtd_report_ir_illegal_access (vtd_as, addr, flags &IOMMU_WO ); + entry.target_as = &address_space_memory; + entry.iova =0 ; + entry.translated_addr =0 ; + entry.addr_mask = ~VTD_PAGE_MASK_4K ; + entry.perm =IOMMU_NONE ; + entry.pasid =PCI_NO_PASID ; + }else { + entry =vtd_iommu_translate (iommu, addr, flags,0 ); + } + +return entry; +} + +static ssize_t vtd_ats_request_translation (PCIBus *bus,void *opaque, +int devfn,uint32_t pasid, +bool priv_req,bool exec_req, +hwaddr addr,size_t length, +bool no_write,IOMMUTLBEntry *result, +size_t result_length, +uint32_t *err_count) +{ +IntelIOMMUState *s = opaque; +VTDAddressSpace *vtd_as; +IOMMUAccessFlags flags =IOMMU_ACCESS_FLAG_FULL (true, !no_write, exec_req, + priv_req, false, false); +ssize_t res_index =0 ; +hwaddr target_address = addr + length; +IOMMUTLBEntry entry; + + vtd_as =vtd_find_add_as (s, bus, devfn, pasid); + *err_count =0 ; + +while ((addr < target_address) && (res_index < result_length)) { + entry =vtd_iommu_ats_do_translate (&vtd_as->iommu, addr, flags); + entry.perm &= ~IOMMU_GLOBAL ;/* Spec 4.1.2: Global Mapping never set */ + +if ((entry.perm & flags) != flags) { + *err_count +=1 ;/* Less than expected */ + } + + result[res_index] = entry; + res_index +=1 ; + addr = (addr & (~entry.addr_mask)) + (entry.addr_mask +1 ); + } + +/* Buffer too small */ +if (addr < target_address) { +return -ENOMEM ; + } + +return res_index; +} +static void vtd_init_iotlb_notifier (PCIBus *bus,void *opaque,int devfn,IOMMUNotifier *n,IOMMUNotify fn,void *user_opaque)
@@ -4787,6 +4849,7 @@ static PCIIOMMUOps vtd_iommu_ops = { .init_iotlb_notifier = vtd_init_iotlb_notifier, .register_iotlb_notifier = vtd_register_iotlb_notifier, .unregister_iotlb_notifier = vtd_unregister_iotlb_notifier, + .ats_request_translation = vtd_ats_request_translation, };static bool vtd_decide_config (IntelIOMMUState *s,Error **errp)
Signature · 1 line
2.49.0