Skip to contentSkip to conversation

x86/msi: dynamically map pages for MSI-X tables

PATCH In review Ruben Hakobyan 2 participants 2 messages Aug 14, 2023 – Aug 16, 2023

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 x86/msi: dynamically map pages for MSI-X tables

Xen reserves a constant number of pages that can be used for mapping MSI-X tables. This limit is defined by FIX_MSIX_MAX_PAGES in fixmap.h.

Reserving a fixed number of pages could result in an -ENOMEM if a device requests a new page when the fixmap limit is exhausted and will necessitate manually adjusting the limit before compilation.

To avoid the issues with the current fixmap implementation, we modify the MSI-X page mapping logic to instead dynamically map new pages when they are needed by making use of ioremap().

Note that this approach is not suitable for 32-bit architectures, where the virtual address space is considerably smaller.

---
Changelog:
* v2:
- Updated the commit message to talk about the potential use on 32-bit architectures
- Addressed the various comments for improving the code

---
 xen/arch/x86/include/asm/fixmap.h |  2 -
 xen/arch/x86/include/asm/msi.h    |  5 +--
 xen/arch/x86/msi.c                | 69 ++++++++-----------------------
 3 files changed, 18 insertions(+), 58 deletions(-)
xen/arch/x86/include/asm/fixmap.h +0 -2
index 516ec3fa6c..139c3e2dcc 100644
@@ -61,8 +61,6 @@ enum fixed_addresses {
     FIX_ACPI_END = FIX_ACPI_BEGIN + NUM_FIXMAP_ACPI_PAGES - 1,
     FIX_HPET_BASE,
     FIX_TBOOT_SHARED_BASE,
-    FIX_MSIX_IO_RESERV_BASE,
-    FIX_MSIX_IO_RESERV_END = FIX_MSIX_IO_RESERV_BASE + FIX_MSIX_MAX_PAGES -1,
     FIX_TBOOT_MAP_ADDRESS,
     FIX_APEI_RANGE_BASE,
     FIX_APEI_RANGE_END = FIX_APEI_RANGE_BASE + FIX_APEI_RANGE_MAX -1,
xen/arch/x86/include/asm/msi.h +1 -4
index a53ade95c9..16c80c9883 100644
@@ -55,9 +55,6 @@
 #define	 MSI_ADDR_DEST_ID_MASK		0x00ff000
 #define  MSI_ADDR_DEST_ID(dest)		(((dest) << MSI_ADDR_DEST_ID_SHIFT) & MSI_ADDR_DEST_ID_MASK)
 
-/* MAX fixed pages reserved for mapping MSIX tables. */
-#define FIX_MSIX_MAX_PAGES              512
-
 struct msi_info {
     pci_sbdf_t sbdf;
     int irq;
@@ -213,7 +210,7 @@ struct arch_msix {
         unsigned long first, last;
     } table, pba;
     int table_refcnt[MAX_MSIX_TABLE_PAGES];
-    int table_idx[MAX_MSIX_TABLE_PAGES];
+    void __iomem *table_va[MAX_MSIX_TABLE_PAGES];
     spinlock_t table_lock;
     bool host_maskall, guest_maskall;
     domid_t warned;
xen/arch/x86/msi.c +17 -52

135 lines not on this page · raw message

Signature · 1 line
2.40.1
#1 lore raw
Quoted 7 lines · Ruben Hakobyan

Xen reserves a constant number of pages that can be used for mapping MSI-X tables. This limit is defined by FIX_MSIX_MAX_PAGES in fixmap.h.

Reserving a fixed number of pages could result in an -ENOMEM if a device requests a new page when the fixmap limit is exhausted and will necessitate manually adjusting the limit before compilation.

Quoted 6 lines

To avoid the issues with the current fixmap implementation, we modify the MSI-X page mapping logic to instead dynamically map new pages when they are needed by making use of ioremap().

Note that this approach is not suitable for 32-bit architectures, where the virtual address space is considerably smaller.

This addresses one of the issues raised on v1. There was also the concern of the mapping now potentially involving memory allocation (if page tables need populating). I wonder whether, alongside emitting a warning, we couldn't fall back to using fixmap in such an event.

Furthermore there's the concern regarding VA space use: If the 512 entries we presently permit don't suffice, we talk about more than 2M worth of mappings, i.e. more than 4M worth of VA space (due to guard pages). That's not entirely negligible, even if still only a fairly small share out of the 64G that we set aside right now, and hence probably wants at least mentioning.

xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 45

Quoted 7 lines
xen/arch/x86/msi.c +2 -2
-static int msix_get_fixmap(struct arch_msix *msix, u64 table_paddr,
+static void __iomem *msix_get_entry(struct arch_msix *msix, u64 table_paddr,
                            u64 entry_paddr)
 {
     long nr_page;
-    int idx;
+    void __iomem *va = NULL;

Unnecessary initializer.

xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 50

Quoted 5 lines
     nr_page = (entry_paddr >> PAGE_SHIFT) - (table_paddr >> PAGE_SHIFT);
 
     if ( nr_page < 0 || nr_page >= MAX_MSIX_TABLE_PAGES )
xen/arch/x86/msi.c +1 -1
-        return -EINVAL;
+        return NULL;

Please don't lose the error code; callers should not need to blindly (and hence possibly wrongly) assume -ENOMEM when getting back NULL. See xen/err.h.

Quoted 13 lines
     spin_lock(&msix->table_lock);
     if ( msix->table_refcnt[nr_page]++ == 0 )
     {
xen/arch/x86/msi.c +2 -4
-        idx = msix_fixmap_alloc();
-        if ( idx < 0 )
+        va = ioremap(round_pgdown(entry_paddr), PAGE_SIZE);
+        if ( va == NULL )
         {
             msix->table_refcnt[nr_page]--;
             goto out;
         }
-        set_fixmap_nocache(idx, entry_paddr);
-        msix->table_idx[nr_page] = idx;

xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 68

Quoted 10 lines
xen/arch/x86/msi.c +3 -2
+        msix->table_va[nr_page] = va;
     }
     else
-        idx = msix->table_idx[nr_page];
+        va = msix->table_va[nr_page];
 
  out:
     spin_unlock(&msix->table_lock);
-    return idx;
+    return va + (entry_paddr & ~PAGE_MASK);

This is wrong in the error case, i.e. when va is NULL.

xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 74

Quoted 7 lines

}

xen/arch/x86/msi.c +2 -1
-static void msix_put_fixmap(struct arch_msix *msix, int idx)
+static void msix_put_entry(struct arch_msix *msix, const void __iomem *entry_va)
 {
     int i;
+    void __iomem *va = (void*)round_pgdown((unsigned long)entry_va);

Nit (style): Blank after * please.

Quoted 9 lines
     spin_lock(&msix->table_lock);
     for ( i = 0; i < MAX_MSIX_TABLE_PAGES; i++ )
     {
xen/arch/x86/msi.c +1 -1
-        if ( msix->table_idx[i] == idx )
+        if ( msix->table_va[i] == va )
             break;
     }
     if ( i == MAX_MSIX_TABLE_PAGES )

xen/arch/x86/msi.c static void msix_put_fixmap(struct arch_msix *msix, int idx) · line 88

Quoted 8 lines
xen/arch/x86/msi.c +2 -3
@@ -115,9 +84,8 @@ static void msix_put_fixmap(struct arch_msix *msix, int idx)
 
     if ( --msix->table_refcnt[i] == 0 )
     {
-        clear_fixmap(idx);
-        msix_fixmap_free(idx);
-        msix->table_idx[i] = 0;
+        iounmap(va);
+        msix->table_va[i] = NULL;

While possibly benign here, please clear the field before unmapping, such that there's no doubt about a transiently dangling pointer.

Jan