Skip to contentSkip to conversation

intel_iommu: Add ATS support

PATCH New CLEMENT MATHIEU--DRIF 1 participant 11 messages Jun 28, 2025

lore
Share & tools
lore
#0 lore raw

v5 cover intel_iommu: Add ATS support

This patch set belongs to a list of series that add SVM support for VT-d. Here we focus on implementing the ATS API exposed through the PCIIOMMUOps callbacks.

This work is based on the VT-d specification version 4.1 (March 2023).

Here is a link to our GitHub repository where you can find:
- Qemu with all the patches for SVM
- ATS
- PRI
- Device IOTLB invalidations
- Requests with already pre-translated addresses
- A demo device
- A simple driver for the demo device
- A userspace program (for testing and demonstration purposes)

https://github.com/BullSequana/Qemu-in-guest-SVM-demo

What is ATS?
''''''''''''

ATS (Address Translation Service) is a PCIe-level protocol that enables PCIe devices to query an IOMMU for virtual to physical address translations in a specific address space (potentially identified by a PASID). When a device receives a translation response from an IOMMU, it may decide to store it in an internal cache, often known as "ATC" (Address Translation Cache) or "Device IOTLB". When triggering a memory operation using a physical address obtained via ATS, the device must set the AT field of the PCIe message to "translated" to prevent the IOMMU from translating the address again. To keep page tables and caches consistent, the host system can ask the IOMMU to send invalidation requests to the devices.

Changelog:

v5
- Migrate to the PCIIOMMUOps API

v4
- Fix compilation issue on 32bit hosts

v3
- Fix compilation issue in non-debug mode (Michael)

v2
- Rebase on master after merge of Zhenzhong's FLTS series
- Rename the series as it is now based on master.

- Changes after review by Michael:
- Split long lines in memory.h
- Change patch encoding (no UTF-8)

- Changes after review by Zhenzhong:
- Rework "Fill the PASID field when creating an IOMMUTLBEntry"

Clement Mathieu--Drif (10):
pci: Add a memory attribute for pre-translated DMA operations
memory: Add permissions in IOMMUAccessFlags
memory: Allow to store the PASID in IOMMUTLBEntry
intel_iommu: Fill the PASID field when creating an IOMMUTLBEntry
intel_iommu: Declare supported PASID size
intel_iommu: Implement vtd_get_iotlb_info from PCIIOMMUOps
intel_iommu: Implement the PCIIOMMUOps callbacks related to
invalidations of device-IOTLB
intel_iommu: Return page walk level even when the translation fails
intel_iommu: Set address mask when a translation fails and adjust W
permission
intel_iommu: Add support for ATS

 hw/i386/intel_iommu.c          | 142 ++++++++++++++++++++++++++++++---
 hw/i386/intel_iommu_internal.h |   1 +
 include/exec/memattrs.h        |   3 +
 include/hw/pci/pci.h           |   9 +++
 include/system/memory.h        |  24 +++++-
 5 files changed, 164 insertions(+), 15 deletions(-)
Signature · 1 line
2.49.0
#1 lore raw

PATCH v5 1/10 pci: Add a memory attribute for pre-translated DMA operations

The address_type bit will be set to PCI_AT_TRANSLATED by devices that use cached addresses obtained via ATS.

---
 include/exec/memattrs.h | 3 +++
 include/hw/pci/pci.h    | 9 +++++++++
 2 files changed, 12 insertions(+)
include/exec/memattrs.h +3 -0
index 8db1d30464..52ee955249 100644
@@ -54,6 +54,9 @@ typedef struct MemTxAttrs {
      */
     unsigned int pid:8;
 
+    /* PCI - IOMMU operations, see PCIAddressType */
+    unsigned int address_type:1;
+
     /*
      * Bus masters which don't specify any attributes will get this
      * (via the MEMTXATTRS_UNSPECIFIED constant), so that we can
include/hw/pci/pci.h +9 -0
index df3cc7b875..6b7d3ac8a3 100644
@@ -134,6 +134,15 @@ struct PCIHostDeviceAddress {
     unsigned int function;
 };
 
+/*
+ * Represents the Address Type (AT) field in a PCI request,
+ * see MemTxAttrs.address_type
+ */
+typedef enum PCIAddressType {
+    PCI_AT_UNTRANSLATED = 0, /* Default when no attribute is set */
+    PCI_AT_TRANSLATED = 1,
+} PCIAddressType;
+
 typedef void PCIConfigWriteFunc(PCIDevice *pci_dev,
                                 uint32_t address, uint32_t data, int len);
 typedef uint32_t PCIConfigReadFunc(PCIDevice *pci_dev,
Signature · 1 line
2.49.0
#2 lore raw

PATCH v5 2/10 memory: Add permissions in IOMMUAccessFlags

In reply to CLEMENT MATHIEU--DRIF

This will be necessary for devices implementing ATS.
We also define a new macro IOMMU_ACCESS_FLAG_FULL in addition to IOMMU_ACCESS_FLAG to support more access flags. IOMMU_ACCESS_FLAG is kept for convenience and backward compatibility.

Here are the flags added (defined by the PCIe 5 specification) :
- Execute Requested
- Privileged Mode Requested
- Global
- Untranslated Only

IOMMU_ACCESS_FLAG sets the additional flags to 0

---
 include/system/memory.h | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)
include/system/memory.h +21 -2
index 46248d4a52..1672622d70 100644
@@ -109,15 +109,34 @@ struct MemoryRegionSection {
 
 typedef struct IOMMUTLBEntry IOMMUTLBEntry;
 
-/* See address_space_translate: bit 0 is read, bit 1 is write.  */
+/*
+ * See address_space_translate:
+ *      - bit 0 : read
+ *      - bit 1 : write
+ *      - bit 2 : exec
+ *      - bit 3 : priv
+ *      - bit 4 : global
+ *      - bit 5 : untranslated only
+ */
 typedef enum {
     IOMMU_NONE = 0,
     IOMMU_RO   = 1,
     IOMMU_WO   = 2,
     IOMMU_RW   = 3,
+    IOMMU_EXEC = 4,
+    IOMMU_PRIV = 8,
+    IOMMU_GLOBAL = 16,
+    IOMMU_UNTRANSLATED_ONLY = 32,
 } IOMMUAccessFlags;
 
-#define IOMMU_ACCESS_FLAG(r, w) (((r) ? IOMMU_RO : 0) | ((w) ? IOMMU_WO : 0))
+#define IOMMU_ACCESS_FLAG(r, w)     (((r) ? IOMMU_RO : 0) | \
+                                    ((w) ? IOMMU_WO : 0))
+#define IOMMU_ACCESS_FLAG_FULL(r, w, x, p, g, uo) \
+                                    (IOMMU_ACCESS_FLAG(r, w) | \
+                                    ((x) ? IOMMU_EXEC : 0) | \
+                                    ((p) ? IOMMU_PRIV : 0) | \
+                                    ((g) ? IOMMU_GLOBAL : 0) | \
+                                    ((uo) ? IOMMU_UNTRANSLATED_ONLY : 0))
 
 struct IOMMUTLBEntry {
     AddressSpace    *target_as;
Signature · 1 line
2.49.0
#3 lore raw

PATCH v5 3/10 memory: Allow to store the PASID in IOMMUTLBEntry

In reply to CLEMENT MATHIEU--DRIF

This will be useful for devices that support ATS and need to store entries in an ATC (device IOTLB).

---
 include/system/memory.h | 1 +
 1 file changed, 1 insertion(+)
include/system/memory.h +1 -0
index 1672622d70..d6d069fd50 100644
@@ -144,6 +144,7 @@ struct IOMMUTLBEntry {
     hwaddr           translated_addr;
     hwaddr           addr_mask;  /* 0xfff = 4k translation */
     IOMMUAccessFlags perm;
+    uint32_t         pasid;
 };
 
 /*
Signature · 1 line
2.49.0
#4 lore raw

PATCH v5 4/10 intel_iommu: Fill the PASID field when creating an IOMMUTLBEntry

In reply to CLEMENT MATHIEU--DRIF

PASID value must be used by devices as a key (or part of a key) when populating their ATC with the IOTLB entries returned by the IOMMU.

---
 hw/i386/intel_iommu.c | 3 +++
 1 file changed, 3 insertions(+)
hw/i386/intel_iommu.c +3 -0
index 69d72ad35c..0fb4350d48 100644
@@ -2503,6 +2503,7 @@ static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s,
                         .translated_addr = 0,
                         .addr_mask = size - 1,
                         .perm = IOMMU_NONE,
+                        .pasid = vtd_as->pasid,
                     },
                 };
                 memory_region_notify_iommu(&vtd_as->iommu, 0, event);
@@ -3090,6 +3091,7 @@ static void do_invalidate_device_tlb(VTDAddressSpace *vtd_dev_as,
     event.entry.iova = addr;
     event.entry.perm = IOMMU_NONE;
     event.entry.translated_addr = 0;
+    event.entry.pasid = vtd_dev_as->pasid;
     memory_region_notify_iommu(&vtd_dev_as->iommu, 0, event);
 }
 
@@ -3672,6 +3674,7 @@ static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
     IOMMUTLBEntry iotlb = {
         /* We'll fill in the rest later. */
         .target_as = &address_space_memory,
+        .pasid = vtd_as->pasid,
     };
     bool success;
 
Signature · 1 line
2.49.0
#5 lore raw

PATCH v5 5/10 intel_iommu: Declare supported PASID size

In reply to CLEMENT MATHIEU--DRIF

the PSS field of the extended capabilities stores the supported PASID size minus 1. This commit adds support for 8bits PASIDs (limited by MemTxAttrs::pid).

---
 hw/i386/intel_iommu.c          | 2 +-
 hw/i386/intel_iommu_internal.h | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
hw/i386/intel_iommu.c +1 -1
index 0fb4350d48..71497f1936 100644
@@ -4590,7 +4590,7 @@ static void vtd_cap_init(IntelIOMMUState *s)
     }
 
     if (s->pasid) {
-        s->ecap |= VTD_ECAP_PASID;
+        s->ecap |= VTD_ECAP_PASID | VTD_ECAP_PSS;
     }
 }
 
hw/i386/intel_iommu_internal.h +1 -0
index e8b211e8b0..360e937989 100644
@@ -192,6 +192,7 @@
 #define VTD_ECAP_SC                 (1ULL << 7)
 #define VTD_ECAP_MHMV               (15ULL << 20)
 #define VTD_ECAP_SRS                (1ULL << 31)
+#define VTD_ECAP_PSS                (7ULL << 35) /* limit: MemTxAttrs::pid */
 #define VTD_ECAP_PASID              (1ULL << 40)
 #define VTD_ECAP_SMTS               (1ULL << 43)
 #define VTD_ECAP_SLTS               (1ULL << 46)
Signature · 1 line
2.49.0
#6 lore raw

PATCH v5 6/10 intel_iommu: Implement vtd_get_iotlb_info from PCIIOMMUOps

In reply to CLEMENT MATHIEU--DRIF
---
 hw/i386/intel_iommu.c | 10 ++++++++++
 1 file changed, 10 insertions(+)
hw/i386/intel_iommu.c +10 -0
index 71497f1936..affa7768e6 100644
@@ -4733,10 +4733,20 @@ static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
     return &vtd_as->as;
 }
 
+static void vtd_get_iotlb_info(void *opaque, uint8_t *addr_width,
+                               uint32_t *min_page_size)
+{
+    IntelIOMMUState *s = opaque;
+
+    *addr_width = s->aw_bits;
+    *min_page_size = VTD_PAGE_SIZE;
+}
+
 static PCIIOMMUOps vtd_iommu_ops = {
     .get_address_space = vtd_host_dma_iommu,
     .set_iommu_device = vtd_dev_set_iommu_device,
     .unset_iommu_device = vtd_dev_unset_iommu_device,
+    .get_iotlb_info = vtd_get_iotlb_info,
 };
 
 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp)
Signature · 1 line
2.49.0
#7 lore raw

PATCH v5 7/10 intel_iommu: Implement the PCIIOMMUOps callbacks related to invalidations of device-IOTLB

In reply to CLEMENT MATHIEU--DRIF
---
 hw/i386/intel_iommu.c | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
hw/i386/intel_iommu.c +35 -0
index affa7768e6..234c452849 100644
@@ -4733,6 +4733,15 @@ static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
     return &vtd_as->as;
 }
 
+static void vtd_init_iotlb_notifier(PCIBus *bus, void *opaque, int devfn,
+                                    IOMMUNotifier *n, IOMMUNotify fn,
+                                    void *user_opaque)
+{
+    n->opaque = user_opaque;
+    iommu_notifier_init(n, fn, IOMMU_NOTIFIER_DEVIOTLB_EVENTS, 0,
+                        HWADDR_MAX, 0);
+}
+
 static void vtd_get_iotlb_info(void *opaque, uint8_t *addr_width,
                                uint32_t *min_page_size)
 {
@@ -4742,11 +4751,37 @@ static void vtd_get_iotlb_info(void *opaque, uint8_t *addr_width,
     *min_page_size = VTD_PAGE_SIZE;
 }
 
+static void vtd_register_iotlb_notifier(PCIBus *bus, void *opaque,
+                                        int devfn, uint32_t pasid,
+                                        IOMMUNotifier *n)
+{
+    IntelIOMMUState *s = opaque;
+    VTDAddressSpace *vtd_as;
+
+    vtd_as = vtd_find_add_as(s, bus, devfn, pasid);
+    memory_region_register_iommu_notifier(MEMORY_REGION(&vtd_as->iommu), n,
+                                          &error_fatal);
+}
+
+static void vtd_unregister_iotlb_notifier(PCIBus *bus, void *opaque,
+                                          int devfn, uint32_t pasid,
+                                          IOMMUNotifier *n)
+{
+    IntelIOMMUState *s = opaque;
+    VTDAddressSpace *vtd_as;
+
+    vtd_as = vtd_find_add_as(s, bus, devfn, pasid);
+    memory_region_unregister_iommu_notifier(MEMORY_REGION(&vtd_as->iommu), n);
+}
+
 static PCIIOMMUOps vtd_iommu_ops = {
     .get_address_space = vtd_host_dma_iommu,
     .set_iommu_device = vtd_dev_set_iommu_device,
     .unset_iommu_device = vtd_dev_unset_iommu_device,
     .get_iotlb_info = vtd_get_iotlb_info,
+    .init_iotlb_notifier = vtd_init_iotlb_notifier,
+    .register_iotlb_notifier = vtd_register_iotlb_notifier,
+    .unregister_iotlb_notifier = vtd_unregister_iotlb_notifier,
 };
 
 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp)
Signature · 1 line
2.49.0
#8 lore raw

PATCH v5 8/10 intel_iommu: Return page walk level even when the translation fails

In reply to CLEMENT MATHIEU--DRIF

We will use this information in vtd_do_iommu_translate to populate the IOMMUTLBEntry and indicate the correct page mask. This prevents ATS devices from sending many useless translation requests when a megapage or gigapage is not present.

---
 hw/i386/intel_iommu.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)
hw/i386/intel_iommu.c +8 -9
index 234c452849..bff307b9bc 100644
@@ -1987,9 +1987,9 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce,
                              uint32_t pasid)
 {
     dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid);
-    uint32_t level = vtd_get_iova_level(s, ce, pasid);
     uint32_t offset;
     uint64_t flpte, flag_ad = VTD_FL_A;
+    *flpte_level = vtd_get_iova_level(s, ce, pasid);
 
     if (!vtd_iova_fl_check_canonical(s, iova, ce, pasid)) {
         error_report_once("%s: detected non canonical IOVA (iova=0x%" PRIx64 ","
@@ -1998,11 +1998,11 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce,
     }
 
     while (true) {
-        offset = vtd_iova_level_offset(iova, level);
+        offset = vtd_iova_level_offset(iova, *flpte_level);
         flpte = vtd_get_pte(addr, offset);
 
         if (flpte == (uint64_t)-1) {
-            if (level == vtd_get_iova_level(s, ce, pasid)) {
+            if (*flpte_level == vtd_get_iova_level(s, ce, pasid)) {
                 /* Invalid programming of pasid-entry */
                 return -VTD_FR_PASID_ENTRY_FSPTPTR_INV;
             } else {
@@ -2028,15 +2028,15 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce,
         if (is_write && !(flpte & VTD_FL_RW)) {
             return -VTD_FR_SM_WRITE;
         }
-        if (vtd_flpte_nonzero_rsvd(flpte, level)) {
+        if (vtd_flpte_nonzero_rsvd(flpte, *flpte_level)) {
             error_report_once("%s: detected flpte reserved non-zero "
                               "iova=0x%" PRIx64 ", level=0x%" PRIx32
                               "flpte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")",
-                              __func__, iova, level, flpte, pasid);
+                              __func__, iova, *flpte_level, flpte, pasid);
             return -VTD_FR_FS_PAGING_ENTRY_RSVD;
         }
 
-        if (vtd_is_last_pte(flpte, level) && is_write) {
+        if (vtd_is_last_pte(flpte, *flpte_level) && is_write) {
             flag_ad |= VTD_FL_D;
         }
 
@@ -2044,14 +2044,13 @@ static int vtd_iova_to_flpte(IntelIOMMUState *s, VTDContextEntry *ce,
             return -VTD_FR_FS_BIT_UPDATE_FAILED;
         }
 
-        if (vtd_is_last_pte(flpte, level)) {
+        if (vtd_is_last_pte(flpte, *flpte_level)) {
             *flptep = flpte;
-            *flpte_level = level;
             return 0;
         }
 
         addr = vtd_get_pte_addr(flpte, aw_bits);
-        level--;
+        (*flpte_level)--;
     }
 }
 
Signature · 1 line
2.49.0
#9 lore raw

PATCH v5 9/10 intel_iommu: Set address mask when a translation fails and adjust W permission

In reply to CLEMENT MATHIEU--DRIF

Implements the behavior defined in section 10.2.3.5 of PCIe spec rev 5. This is needed by devices that support ATS.

---
 hw/i386/intel_iommu.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)
hw/i386/intel_iommu.c +9 -3
index bff307b9bc..1b1b0b5632 100644
@@ -2091,7 +2091,8 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
     uint8_t bus_num = pci_bus_num(bus);
     VTDContextCacheEntry *cc_entry;
     uint64_t pte, page_mask;
-    uint32_t level, pasid = vtd_as->pasid;
+    uint32_t level = UINT32_MAX;
+    uint32_t pasid = vtd_as->pasid;
     uint16_t source_id = PCI_BUILD_BDF(bus_num, devfn);
     int ret_fr;
     bool is_fpd_set = false;
@@ -2250,14 +2251,19 @@ out:
     entry->iova = addr & page_mask;
     entry->translated_addr = vtd_get_pte_addr(pte, s->aw_bits) & page_mask;
     entry->addr_mask = ~page_mask;
-    entry->perm = access_flags;
+    entry->perm = (is_write ? access_flags : (access_flags & (~IOMMU_WO)));
     return true;
 
 error:
     vtd_iommu_unlock(s);
     entry->iova = 0;
     entry->translated_addr = 0;
-    entry->addr_mask = 0;
+    /*
+     * Set the mask for ATS (the range must be present even when the
+     * translation fails : PCIe rev 5 10.2.3.5)
+     */
+    entry->addr_mask = (level != UINT32_MAX) ?
+                       (~vtd_pt_level_page_mask(level)) : (~VTD_PAGE_MASK_4K);
     entry->perm = IOMMU_NONE;
     return false;
 }
Signature · 1 line
2.49.0
#10 lore raw

PATCH v5 10/10 intel_iommu: Add support for ATS

In reply to CLEMENT MATHIEU--DRIF
---
 hw/i386/intel_iommu.c | 63 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
hw/i386/intel_iommu.c +63 -0
index 1b1b0b5632..fe9a5f2872 100644
@@ -4738,6 +4738,68 @@ static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
     return &vtd_as->as;
 }
 
+static IOMMUTLBEntry vtd_iommu_ats_do_translate(IOMMUMemoryRegion *iommu,
+                                                hwaddr addr,
+                                                IOMMUAccessFlags flags)
+{
+    IOMMUTLBEntry entry;
+    VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
+
+    if (vtd_is_interrupt_addr(addr)) {
+        vtd_report_ir_illegal_access(vtd_as, addr, flags & IOMMU_WO);
+        entry.target_as = &address_space_memory;
+        entry.iova = 0;
+        entry.translated_addr = 0;
+        entry.addr_mask = ~VTD_PAGE_MASK_4K;
+        entry.perm = IOMMU_NONE;
+        entry.pasid = PCI_NO_PASID;
+    } else {
+        entry = vtd_iommu_translate(iommu, addr, flags, 0);
+    }
+
+    return entry;
+}
+
+static ssize_t vtd_ats_request_translation(PCIBus *bus, void *opaque,
+                                           int devfn, uint32_t pasid,
+                                           bool priv_req, bool exec_req,
+                                           hwaddr addr, size_t length,
+                                           bool no_write, IOMMUTLBEntry *result,
+                                           size_t result_length,
+                                           uint32_t *err_count)
+{
+    IntelIOMMUState *s = opaque;
+    VTDAddressSpace *vtd_as;
+    IOMMUAccessFlags flags = IOMMU_ACCESS_FLAG_FULL(true, !no_write, exec_req,
+                                                    priv_req, false, false);
+    ssize_t res_index = 0;
+    hwaddr target_address = addr + length;
+    IOMMUTLBEntry entry;
+
+    vtd_as = vtd_find_add_as(s, bus, devfn, pasid);
+    *err_count = 0;
+
+    while ((addr < target_address) && (res_index < result_length)) {
+        entry = vtd_iommu_ats_do_translate(&vtd_as->iommu, addr, flags);
+        entry.perm &= ~IOMMU_GLOBAL; /* Spec 4.1.2: Global Mapping never set */
+
+        if ((entry.perm & flags) != flags) {
+            *err_count += 1; /* Less than expected */
+        }
+
+        result[res_index] = entry;
+        res_index += 1;
+        addr = (addr & (~entry.addr_mask)) + (entry.addr_mask + 1);
+    }
+
+    /* Buffer too small */
+    if (addr < target_address) {
+        return -ENOMEM;
+    }
+
+    return res_index;
+}
+
 static void vtd_init_iotlb_notifier(PCIBus *bus, void *opaque, int devfn,
                                     IOMMUNotifier *n, IOMMUNotify fn,
                                     void *user_opaque)
@@ -4787,6 +4849,7 @@ static PCIIOMMUOps vtd_iommu_ops = {
     .init_iotlb_notifier = vtd_init_iotlb_notifier,
     .register_iotlb_notifier = vtd_register_iotlb_notifier,
     .unregister_iotlb_notifier = vtd_unregister_iotlb_notifier,
+    .ats_request_translation = vtd_ats_request_translation,
 };
 
 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp)
Signature · 1 line
2.49.0