x86/msi: dynamically map pages for MSI-X tables
PATCH v2 1/1 x86/msi: dynamically map pages for MSI-X tables
Xen reserves a constant number of pages that can be used for mapping MSI-X tables. This limit is defined by FIX_MSIX_MAX_PAGES in fixmap.h.
Reserving a fixed number of pages could result in an -ENOMEM if a device requests a new page when the fixmap limit is exhausted and will necessitate manually adjusting the limit before compilation.
To avoid the issues with the current fixmap implementation, we modify the MSI-X page mapping logic to instead dynamically map new pages when they are needed by making use of ioremap().
Note that this approach is not suitable for 32-bit architectures, where the virtual address space is considerably smaller.
---
Changelog:
* v2:
- Updated the commit message to talk about the potential use on 32-bit architectures
- Addressed the various comments for improving the code
--- xen/arch/x86/include/asm/fixmap.h | 2 - xen/arch/x86/include/asm/msi.h | 5 +-- xen/arch/x86/msi.c | 69 ++++++++----------------------- 3 files changed, 18 insertions(+), 58 deletions(-)
xen/arch/x86/include/asm/fixmap.h +0 -2
@@ -61,8 +61,6 @@ enum fixed_addresses {FIX_ACPI_END =FIX_ACPI_BEGIN +NUM_FIXMAP_ACPI_PAGES -1 ,FIX_HPET_BASE ,FIX_TBOOT_SHARED_BASE , -FIX_MSIX_IO_RESERV_BASE , -FIX_MSIX_IO_RESERV_END =FIX_MSIX_IO_RESERV_BASE +FIX_MSIX_MAX_PAGES -1 ,FIX_TBOOT_MAP_ADDRESS ,FIX_APEI_RANGE_BASE , FIX_APEI_RANGE_END = FIX_APEI_RANGE_BASE + FIX_APEI_RANGE_MAX -1,
xen/arch/x86/include/asm/msi.h +1 -4
@@ -55,9 +55,6 @@#define MSI_ADDR_DEST_ID_MASK 0x00ff000#define MSI_ADDR_DEST_ID (dest) (((dest) << MSI_ADDR_DEST_ID_SHIFT) & MSI_ADDR_DEST_ID_MASK) -/* MAX fixed pages reserved for mapping MSIX tables. */ -#define FIX_MSIX_MAX_PAGES 512 -struct msi_info {pci_sbdf_t sbdf;int irq;
@@ -213,7 +210,7 @@ struct arch_msix {unsigned long first, last; } table, pba;int table_refcnt[MAX_MSIX_TABLE_PAGES ]; -int table_idx[MAX_MSIX_TABLE_PAGES ]; +void __iomem *table_va[MAX_MSIX_TABLE_PAGES ];spinlock_t table_lock;bool host_maskall, guest_maskall;domid_t warned;
xen/arch/x86/msi.c +17 -52
135 lines not on this page · raw message
Signature · 1 line
2.40.1
Quoted 7 lines · Ruben Hakobyan
Xen reserves a constant number of pages that can be used for mapping MSI-X tables. This limit is defined by
FIX_MSIX_MAX_PAGESinfixmap.h.Reserving a fixed number of pages could result in an
-ENOMEMif a device requests a new page when the fixmap limit is exhausted and will necessitate manually adjusting the limit before compilation.
Quoted 6 lines
To avoid the issues with the current fixmap implementation, we modify the MSI-X page mapping logic to instead dynamically map new pages when they are needed by making use of
ioremap().Note that this approach is not suitable for 32-bit architectures, where the virtual address space is considerably smaller.
This addresses one of the issues raised on v1. There was also the concern of the mapping now potentially involving memory allocation (if page tables need populating). I wonder whether, alongside emitting a warning, we couldn't fall back to using fixmap in such an event.
Furthermore there's the concern regarding VA space use: If the 512 entries we presently permit don't suffice, we talk about more than 2M worth of mappings, i.e. more than 4M worth of VA space (due to guard pages). That's not entirely negligible, even if still only a fairly small share out of the 64G that we set aside right now, and hence probably wants at least mentioning.
xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 45
Quoted 7 lines
xen/arch/x86/msi.c +2
-2-static int msix_get_fixmap (struct arch_msix *msix,u64 table_paddr, +static void __iomem *msix_get_entry (struct arch_msix *msix,u64 table_paddr,u64 entry_paddr) {long nr_page; -int idx; +void __iomem *va =NULL ;
Unnecessary initializer.
xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 50
Quoted 5 lines
nr_page = (entry_paddr >> PAGE_SHIFT) - (table_paddr >> PAGE_SHIFT); if ( nr_page < 0 || nr_page >= MAX_MSIX_TABLE_PAGES )xen/arch/x86/msi.c +1
-1-return -EINVAL ; +return NULL ;
Please don't lose the error code; callers should not need to blindly (and hence possibly wrongly) assume -ENOMEM when getting back NULL. See xen/err.h.
Quoted 13 lines
spin_lock(&msix->table_lock); if ( msix->table_refcnt[nr_page]++ == 0 ) {xen/arch/x86/msi.c +2
-4- idx =msix_fixmap_alloc (); -if ( idx <0 ) + va =ioremap (round_pgdown (entry_paddr),PAGE_SIZE ); +if ( va ==NULL ) { msix->table_refcnt[nr_page]--; goto out; } -set_fixmap_nocache (idx, entry_paddr); - msix->table_idx[nr_page] = idx;
xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 68
Quoted 10 lines
xen/arch/x86/msi.c +3
-2+ msix->table_va[nr_page] = va; }else - idx = msix->table_idx[nr_page]; + va = msix->table_va[nr_page]; out:spin_unlock (&msix->table_lock); -return idx; +return va + (entry_paddr & ~PAGE_MASK );
This is wrong in the error case, i.e. when va is NULL.
xen/arch/x86/msi.c boolean_param("msi", use_msi); · line 74
Quoted 7 lines
}
xen/arch/x86/msi.c +2
-1-static void msix_put_fixmap (struct arch_msix *msix,int idx) +static void msix_put_entry (struct arch_msix *msix,const void __iomem *entry_va) {int i; +void __iomem *va = (void *)round_pgdown ((unsigned long )entry_va);
Nit (style): Blank after * please.
Quoted 9 lines
spin_lock(&msix->table_lock); for ( i = 0; i < MAX_MSIX_TABLE_PAGES; i++ ) {xen/arch/x86/msi.c +1
-1-if ( msix->table_idx[i] == idx ) +if ( msix->table_va[i] == va )break ; }if ( i ==MAX_MSIX_TABLE_PAGES )
xen/arch/x86/msi.c static void msix_put_fixmap(struct arch_msix *msix, int idx) · line 88
Quoted 8 lines
xen/arch/x86/msi.c +2
-3@@ -115,9 +84,8 @@ static void msix_put_fixmap(struct arch_msix *msix, int idx)if ( --msix->table_refcnt[i] ==0 ) { -clear_fixmap (idx); -msix_fixmap_free (idx); - msix->table_idx[i] =0 ; +iounmap (va); + msix->table_va[i] =NULL ;
While possibly benign here, please clear the field before unmapping, such that there's no doubt about a transiently dangling pointer.
Jan