Page table handling updates for AMD-IOMMU (AMD-Vi)
cover Page table handling updates for AMD-IOMMU (AMD-Vi)
Hi,
this series of patches enhance the page table handling code for AMD-IOMMU (AMD-Vi). With these patches the page table handling code is able to map a full 64 bit device address space. The old limit was a 39 bit address space.
This series also contains initial patches for upcoming large page support in the AMD-Vi driver. Full large page support requires changes to the IOMMU-API which are not part of this patch set.
Joerg
Diffstat:
arch/x86/include/asm/amd_iommu_types.h | 38 +++--- arch/x86/kernel/amd_iommu.c | 242 ++++++++++++++++++++------------ 2 files changed, 170 insertions(+), 110 deletions(-)
Shortlog:
Joerg Roedel (14):
x86/amd-iommu: Make fetch_pte aware of dynamic mapping levels x86/amd-iommu: Use fetch_pte in iommu_unmap_page x86/amd-iommu: Use fetch_pte in amd_iommu_iova_to_phys x86/amd-iommu: Add a gneric version of amd_iommu_flush_all_devices x86/amd-iommu: Introduce set_dte_entry function x86/amd-iommu: Flush domains if address space size was increased x86/amd-iommu: Introduce increase_address_space function x86/amd-iommu: Change alloc_pte to support 64 bit address space x86/amd-iommu: Remove last usages of IOMMU_PTE_L0_INDEX x86/amd-iommu: Remove bus_addr check in iommu_map_page x86/amd-iommu: Use 2-level page tables for dma_ops domains x86/amd-iommu: Remove old page table handling macros x86/amd-iommu: Support higher level PTEs in iommu_page_unmap x86/amd-iommu: Change iommu_map_page to support multiple page sizes
PATCH 1/14 x86/amd-iommu: Make fetch_pte aware of dynamic mapping levels
This patch changes the fetch_pte function in the AMD IOMMU driver to support dynamic mapping levels.
--- arch/x86/include/asm/amd_iommu_types.h | 9 +++++++++ arch/x86/kernel/amd_iommu.c | 24 +++++++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +9 -0
@@ -146,12 +146,21 @@#define PAGE_MODE_1_LEVEL 0x01#define PAGE_MODE_2_LEVEL 0x02#define PAGE_MODE_3_LEVEL 0x03 +#define PAGE_MODE_4_LEVEL 0x04 +#define PAGE_MODE_5_LEVEL 0x05 +#define PAGE_MODE_6_LEVEL 0x06#define IOMMU_PDE_NL_0 0x000ULL#define IOMMU_PDE_NL_1 0x200ULL#define IOMMU_PDE_NL_2 0x400ULL#define IOMMU_PDE_NL_3 0x600ULL +#define PM_LEVEL_SHIFT (x) (12 + ((x) * 9)) +#define PM_LEVEL_SIZE (x) (((x) < 6) ? \ + ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ + (0xffffffffffffffffULL)) +#define PM_LEVEL_INDEX (x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL) +#define IOMMU_PTE_L2_INDEX (address) (((address) >> 30) & 0x1ffULL)#define IOMMU_PTE_L1_INDEX (address) (((address) >> 21) & 0x1ffULL)#define IOMMU_PTE_L0_INDEX (address) (((address) >> 12) & 0x1ffULL)
arch/x86/kernel/amd_iommu.c +13 -11
@@ -61,6 +61,8 @@ static u64* alloc_pte(struct protection_domain *dom,static void dma_ops_reserve_addresses (struct dma_ops_domain *dom,unsigned long start_page,unsigned int pages); +static u64 *fetch_pte (struct protection_domain *domain, +unsigned long address);#ifndef BUS_NOTIFY_UNBOUND_DRIVER #define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
@@ -670,24 +672,24 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,* This function checks if there is a PTE for a given dma address. If * there is one, it returns the pointer to it. */ -static u64 *fetch_pte (struct protection_domain *domain, +static u64 *fetch_pte (struct protection_domain *domain,unsigned long address) { +int level;u64 *pte; - pte = &domain->pt_root[IOMMU_PTE_L2_INDEX (address)]; + level = domain->mode -1 ; + pte = &domain->pt_root[PM_LEVEL_INDEX (level, address)]; -if (!IOMMU_PTE_PRESENT (*pte)) -return NULL ; +while (level >0 ) { +if (!IOMMU_PTE_PRESENT (*pte)) +return NULL ; - pte =IOMMU_PTE_PAGE (*pte); - pte = &pte[IOMMU_PTE_L1_INDEX (address)]; + level -=1 ; -if (!IOMMU_PTE_PRESENT (*pte)) -return NULL ; - - pte =IOMMU_PTE_PAGE (*pte); - pte = &pte[IOMMU_PTE_L0_INDEX (address)]; + pte =IOMMU_PTE_PAGE (*pte); + pte = &pte[PM_LEVEL_INDEX (level, address)]; + }return pte; }
Signature · 1 line
1.6.3.3
PATCH 2/14 x86/amd-iommu: Use fetch_pte in iommu_unmap_page
In reply to Joerg RoedelInstead of reimplementing existing logic use fetch_pte to walk the page table in iommu_unmap_page.
--- arch/x86/kernel/amd_iommu.c | 19 +++---------------- 1 files changed, 3 insertions(+), 16 deletions(-)
arch/x86/kernel/amd_iommu.c +3 -16
@@ -546,23 +546,10 @@ static int iommu_map_page(struct protection_domain *dom,static void iommu_unmap_page (struct protection_domain *dom,unsigned long bus_addr) { -u64 *pte; - - pte = &dom->pt_root[IOMMU_PTE_L2_INDEX (bus_addr)]; - -if (!IOMMU_PTE_PRESENT (*pte)) -return ; - - pte =IOMMU_PTE_PAGE (*pte); - pte = &pte[IOMMU_PTE_L1_INDEX (bus_addr)]; - -if (!IOMMU_PTE_PRESENT (*pte)) -return ; - - pte =IOMMU_PTE_PAGE (*pte); - pte = &pte[IOMMU_PTE_L1_INDEX (bus_addr)]; +u64 *pte =fetch_pte (dom, bus_addr); - *pte =0 ; +if (pte) + *pte =0 ; }/*
Signature · 1 line
1.6.3.3
PATCH 3/14 x86/amd-iommu: Use fetch_pte in amd_iommu_iova_to_phys
In reply to Joerg RoedelDon't reimplement the page table walker in this function. Use the generic one.
--- arch/x86/kernel/amd_iommu.c | 16 ++-------------- 1 files changed, 2 insertions(+), 14 deletions(-)
arch/x86/kernel/amd_iommu.c +2 -14
@@ -2140,21 +2140,9 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,phys_addr_t paddr;u64 *pte; - pte = &domain->pt_root[IOMMU_PTE_L2_INDEX (iova)]; + pte =fetch_pte (domain, iova); -if (!IOMMU_PTE_PRESENT (*pte)) -return 0 ; - - pte =IOMMU_PTE_PAGE (*pte); - pte = &pte[IOMMU_PTE_L1_INDEX (iova)]; - -if (!IOMMU_PTE_PRESENT (*pte)) -return 0 ; - - pte =IOMMU_PTE_PAGE (*pte); - pte = &pte[IOMMU_PTE_L0_INDEX (iova)]; - -if (!IOMMU_PTE_PRESENT (*pte)) +if (!pte || !IOMMU_PTE_PRESENT (*pte))return 0 ; paddr = *pte &IOMMU_PAGE_MASK ;
Signature · 1 line
1.6.3.3
PATCH 4/14 x86/amd-iommu: Add a gneric version of amd_iommu_flush_all_devices
In reply to Joerg RoedelThis patch adds a generic variant of amd_iommu_flush_all_devices function which flushes only the DTEs for a given protection domain.
--- arch/x86/kernel/amd_iommu.c | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/amd_iommu.c +8 -2
@@ -481,13 +481,14 @@ void amd_iommu_flush_all_domains(void) } } -void amd_iommu_flush_all_devices (void ) +static void flush_devices_by_domain (struct protection_domain *domain) {struct amd_iommu *iommu;int i;for (i =0 ; i <= amd_iommu_last_bdf; ++i) { -if (amd_iommu_pd_table[i] ==NULL ) +if ((domain ==NULL && amd_iommu_pd_table[i] ==NULL ) || + (amd_iommu_pd_table[i] != domain))continue ; iommu = amd_iommu_rlookup_table[i];
@@ -499,6 +500,11 @@ void amd_iommu_flush_all_devices(void) } } +void amd_iommu_flush_all_devices (void ) +{ +flush_devices_by_domain (NULL ); +} +/**************************************************************************** * * The functions below are used the create the page table mappings for
Signature · 1 line
1.6.3.3
PATCH 5/14 x86/amd-iommu: Introduce set_dte_entry function
In reply to Joerg RoedelThis function factors out some logic of attach_device to a seperate function. This new function will be used to update device table entries when necessary.
--- arch/x86/kernel/amd_iommu.c | 27 +++++++++++++++++---------- 1 files changed, 17 insertions(+), 10 deletions(-)
arch/x86/kernel/amd_iommu.c +17 -10
@@ -1058,18 +1058,10 @@ static struct protection_domain *domain_for_device(u16 devid)return dom; } -/* -* If a device is not yet associated with a domain, this function does -* assigns it visible for the hardware -*/ -static void attach_device (struct amd_iommu *iommu, -struct protection_domain *domain, -u16 devid) +static void set_dte_entry (u16 devid,struct protection_domain *domain) { -unsigned long flags;u64 pte_root =virt_to_phys (domain->pt_root); - - domain->dev_cnt +=1 ; +unsigned long flags; pte_root |= (domain->mode &DEV_ENTRY_MODE_MASK ) <<DEV_ENTRY_MODE_SHIFT ;
@@ -1082,6 +1074,21 @@ static void attach_device(struct amd_iommu *iommu, amd_iommu_pd_table[devid] = domain;write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); +} + +/* +* If a device is not yet associated with a domain, this function does +* assigns it visible for the hardware +*/ +static void attach_device (struct amd_iommu *iommu, +struct protection_domain *domain, +u16 devid) +{ +/* set the DTE entry */ +set_dte_entry (devid, domain); + +/* increase reference counter */ + domain->dev_cnt +=1 ;/* * We might boot into a crash-kernel here. The crashed kernel
Signature · 1 line
1.6.3.3
PATCH 6/14 x86/amd-iommu: Flush domains if address space size was increased
In reply to Joerg RoedelThist patch introduces the update_domain function which propagates the larger address space of a protection domain to the device table and flushes all relevant DTEs and the domain TLB.
--- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 0 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +1 -0
@@ -235,6 +235,7 @@ struct protection_domain {int mode;/* paging mode (0-6 levels) */ u64 *pt_root;/* page table root pointer */ unsigned long flags;/* flags to find out type of domain */ +bool updated;/* complete domain flush required */ unsigned dev_cnt;/* devices assigned to this domain */ void *priv;/* private data */ };
arch/x86/kernel/amd_iommu.c +32 -0
@@ -63,6 +63,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, unsigned int pages);static u64 *fetch_pte (struct protection_domain *domain,unsigned long address); +static void update_domain (struct protection_domain *domain);#ifndef BUS_NOTIFY_UNBOUND_DRIVER #define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
@@ -546,6 +547,8 @@ static int iommu_map_page(struct protection_domain *dom, *pte = __pte; +update_domain (dom); +return 0 ; }
@@ -762,9 +765,13 @@ static int alloc_new_range(struct amd_iommu *iommu,dma_ops_reserve_addresses (dma_dom, i <<PAGE_SHIFT ,1 ); } +update_domain (&dma_dom->domain); +return 0 ; out_free: +update_domain (&dma_dom->domain); +free_page ((unsigned long )dma_dom->aperture[index]->bitmap);kfree (dma_dom->aperture[index]);
@@ -1294,6 +1301,29 @@ static int get_device_resources(struct device *dev,return 1 ; } +static void update_device_table (struct protection_domain *domain) +{ +int i; + +for (i =0 ; i <= amd_iommu_last_bdf; ++i) { +if (amd_iommu_pd_table[i] != domain) +continue ; +set_dte_entry (i, domain); + } +} + +static void update_domain (struct protection_domain *domain) +{ +if (!domain->updated) +return ; + +update_device_table (domain); +flush_devices_by_domain (domain); +iommu_flush_domain (domain->id); + + domain->updated = false; +} +/* * If the pte_page is not yet allocated this function is called */
@@ -1351,6 +1381,8 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom, }else pte +=IOMMU_PTE_L0_INDEX (address); +update_domain (&dom->domain); +return pte; }
Signature · 1 line
1.6.3.3
PATCH 7/14 x86/amd-iommu: Introduce increase_address_space function
In reply to Joerg RoedelThis function will be used to increase the address space size of a protection domain.
--- arch/x86/include/asm/amd_iommu_types.h | 4 ++++ arch/x86/kernel/amd_iommu.c | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 0 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +4 -0
@@ -160,6 +160,10 @@ ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ (0xffffffffffffffffULL))#define PM_LEVEL_INDEX (x, a) (((a) >> PM_LEVEL_SHIFT((x))) & 0x1ffULL) +#define PM_LEVEL_ENC (x) (((x) << 9) & 0xe00ULL) +#define PM_LEVEL_PDE (x, a) ((a) | PM_LEVEL_ENC((x)) | \ + IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) +#define IOMMU_PTE_L2_INDEX (address) (((address) >> 30) & 0x1ffULL)#define IOMMU_PTE_L1_INDEX (address) (((address) >> 21) & 0x1ffULL)
arch/x86/kernel/amd_iommu.c +27 -0
@@ -1325,6 +1325,33 @@ static void update_domain(struct protection_domain *domain) }/* +* This function is used to add another level to an IO page table. Adding +* another level increases the size of the address space by 9 bits to a size up +* to 64 bits. +*/ +static bool increase_address_space (struct protection_domain *domain, +gfp_t gfp) +{ +u64 *pte; + +if (domain->mode ==PAGE_MODE_6_LEVEL ) +/* address space already 64 bit large */ +return false; + + pte = (void *)get_zeroed_page (gfp); +if (!pte) +return false; + + *pte =PM_LEVEL_PDE (domain->mode, +virt_to_phys (domain->pt_root)); + domain->pt_root = pte; + domain->mode +=1 ; + domain->updated = true; + +return true; +} + +/* * If the pte_page is not yet allocated this function is called */ static u64* alloc_pte(struct protection_domain *dom,
Signature · 1 line
1.6.3.3
PATCH 8/14 x86/amd-iommu: Change alloc_pte to support 64 bit address space
In reply to Joerg RoedelThis patch changes the alloc_pte function to be able to map pages into the whole 64 bit address space supported by AMD IOMMU hardware from the old limit of 2**39 bytes.
--- arch/x86/kernel/amd_iommu.c | 44 +++++++++++++++++++----------------------- 1 files changed, 20 insertions(+), 24 deletions(-)
arch/x86/kernel/amd_iommu.c +20 -24
@@ -55,7 +55,7 @@ struct iommu_cmd {static int dma_ops_unity_map (struct dma_ops_domain *dma_dom,struct unity_map_entry *e);static struct dma_ops_domain *find_protection_domain (u16 devid); -static u64 *alloc_pte (struct protection_domain *dom, +static u64 *alloc_pte (struct protection_domain *domain,unsigned long address,u64 **pte_page,gfp_t gfp); static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
@@ -1351,39 +1351,35 @@ static bool increase_address_space(struct protection_domain *domain,return true; } -/* -* If the pte_page is not yet allocated this function is called -*/ -static u64 *alloc_pte (struct protection_domain *dom, +static u64 *alloc_pte (struct protection_domain *domain,unsigned long address,u64 **pte_page,gfp_t gfp) {u64 *pte, *page; +int level; - pte = &dom->pt_root[IOMMU_PTE_L2_INDEX (address)]; +while (address >PM_LEVEL_SIZE (domain->mode)) +increase_address_space (domain, gfp); -if (!IOMMU_PTE_PRESENT (*pte)) { - page = (u64 *)get_zeroed_page (gfp); -if (!page) -return NULL ; - *pte =IOMMU_L2_PDE (virt_to_phys (page)); - } + level = domain->mode -1 ; + pte = &domain->pt_root[PM_LEVEL_INDEX (level, address)]; - pte =IOMMU_PTE_PAGE (*pte); - pte = &pte[IOMMU_PTE_L1_INDEX (address)]; +while (level >0 ) { +if (!IOMMU_PTE_PRESENT (*pte)) { + page = (u64 *)get_zeroed_page (gfp); +if (!page) +return NULL ; + *pte =PM_LEVEL_PDE (level,virt_to_phys (page)); + } -if (!IOMMU_PTE_PRESENT (*pte)) { - page = (u64 *)get_zeroed_page (gfp); -if (!page) -return NULL ; - *pte =IOMMU_L1_PDE (virt_to_phys (page)); - } + level -=1 ; - pte =IOMMU_PTE_PAGE (*pte); + pte =IOMMU_PTE_PAGE (*pte); -if (pte_page) - *pte_page = pte; +if (pte_page && level ==0 ) + *pte_page = pte; - pte = &pte[IOMMU_PTE_L0_INDEX (address)]; + pte = &pte[PM_LEVEL_INDEX (level, address)]; + }return pte; }
Signature · 1 line
1.6.3.3
PATCH 9/14 x86/amd-iommu: Remove last usages of IOMMU_PTE_L0_INDEX
In reply to Joerg RoedelThis change allows to remove these old macros later.
--- arch/x86/kernel/amd_iommu.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/amd_iommu.c +2 -2
@@ -1402,7 +1402,7 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom, pte =alloc_pte (&dom->domain, address, &pte_page,GFP_ATOMIC ); aperture->pte_pages[APERTURE_PAGE_INDEX (address)] = pte_page; }else - pte +=IOMMU_PTE_L0_INDEX (address); + pte +=PM_LEVEL_INDEX (0 , address);update_domain (&dom->domain);
@@ -1466,7 +1466,7 @@ static void dma_ops_domain_unmap(struct amd_iommu *iommu,if (!pte)return ; - pte +=IOMMU_PTE_L0_INDEX (address); + pte +=PM_LEVEL_INDEX (0 , address);WARN_ON (!*pte);
Signature · 1 line
1.6.3.3
PATCH 10/14 x86/amd-iommu: Remove bus_addr check in iommu_map_page
In reply to Joerg RoedelThe driver now supports full 64 bit device address spaces. So this check is not longer required.
--- arch/x86/kernel/amd_iommu.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-)
arch/x86/kernel/amd_iommu.c +1 -2
@@ -530,8 +530,7 @@ static int iommu_map_page(struct protection_domain *dom, bus_addr =PAGE_ALIGN (bus_addr); phys_addr =PAGE_ALIGN (phys_addr); -/* only support 512GB address spaces for now */ -if (bus_addr >IOMMU_MAP_SIZE_L3 || !(prot &IOMMU_PROT_MASK )) +if (!(prot &IOMMU_PROT_MASK ))return -EINVAL ; pte =alloc_pte (dom, bus_addr,NULL ,GFP_KERNEL );
Signature · 1 line
1.6.3.3
PATCH 11/14 x86/amd-iommu: Use 2-level page tables for dma_ops domains
In reply to Joerg RoedelThe driver now supports a dynamic number of levels for IO page tables. This allows to reduce the number of levels for dma_ops domains by one because a dma_ops domain has usually an address space size between 128MB and 4G.
--- arch/x86/kernel/amd_iommu.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
arch/x86/kernel/amd_iommu.c +1 -1
@@ -1010,7 +1010,7 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu) dma_dom->domain.id =domain_id_alloc ();if (dma_dom->domain.id ==0 ) goto free_dma_dom; - dma_dom->domain.mode =PAGE_MODE_3_LEVEL ; + dma_dom->domain.mode =PAGE_MODE_2_LEVEL ; dma_dom->domain.pt_root = (void *)get_zeroed_page (GFP_KERNEL ); dma_dom->domain.flags =PD_DMA_OPS_MASK ; dma_dom->domain.priv = dma_dom;
Signature · 1 line
1.6.3.3
PATCH 12/14 x86/amd-iommu: Remove old page table handling macros
In reply to Joerg RoedelThese macros are not longer required. So remove them.
--- arch/x86/include/asm/amd_iommu_types.h | 19 ------------------- 1 files changed, 0 insertions(+), 19 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +0 -19
@@ -150,11 +150,6 @@#define PAGE_MODE_5_LEVEL 0x05#define PAGE_MODE_6_LEVEL 0x06 -#define IOMMU_PDE_NL_0 0x000ULL -#define IOMMU_PDE_NL_1 0x200ULL -#define IOMMU_PDE_NL_2 0x400ULL -#define IOMMU_PDE_NL_3 0x600ULL -#define PM_LEVEL_SHIFT (x) (12 + ((x) * 9))#define PM_LEVEL_SIZE (x) (((x) < 6) ? \ ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \
@@ -164,15 +159,6 @@#define PM_LEVEL_PDE (x, a) ((a) | PM_LEVEL_ENC((x)) | \ IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) - -#define IOMMU_PTE_L2_INDEX (address) (((address) >> 30) & 0x1ffULL) -#define IOMMU_PTE_L1_INDEX (address) (((address) >> 21) & 0x1ffULL) -#define IOMMU_PTE_L0_INDEX (address) (((address) >> 12) & 0x1ffULL) - -#define IOMMU_MAP_SIZE_L1 (1ULL << 21) -#define IOMMU_MAP_SIZE_L2 (1ULL << 30) -#define IOMMU_MAP_SIZE_L3 (1ULL << 39) -#define IOMMU_PTE_P (1ULL << 0)#define IOMMU_PTE_TV (1ULL << 1)#define IOMMU_PTE_U (1ULL << 59)
@@ -180,11 +166,6 @@#define IOMMU_PTE_IR (1ULL << 61)#define IOMMU_PTE_IW (1ULL << 62) -#define IOMMU_L1_PDE (address) \ - ((address) | IOMMU_PDE_NL_1 | IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) -#define IOMMU_L2_PDE (address) \ - ((address) | IOMMU_PDE_NL_2 | IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) -#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)#define IOMMU_PTE_PRESENT (pte) ((pte) & IOMMU_PTE_P)#define IOMMU_PTE_PAGE (pte) (phys_to_virt((pte) & IOMMU_PAGE_MASK))
Signature · 1 line
1.6.3.3
PATCH 13/14 x86/amd-iommu: Support higher level PTEs in iommu_page_unmap
In reply to Joerg RoedelThis patch changes fetch_pte and iommu_page_unmap to support different page sizes too.
--- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +1 -0
@@ -158,6 +158,7 @@#define PM_LEVEL_ENC (x) (((x) << 9) & 0xe00ULL)#define PM_LEVEL_PDE (x, a) ((a) | PM_LEVEL_ENC((x)) | \ IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW) +#define PM_PTE_LEVEL (pte) (((pte) >> 9) & 0x7ULL)#define IOMMU_PTE_P (1ULL << 0)#define IOMMU_PTE_TV (1ULL << 1)
arch/x86/kernel/amd_iommu.c +13 -8
@@ -62,7 +62,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,unsigned long start_page, unsigned int pages);static u64 *fetch_pte (struct protection_domain *domain, -unsigned long address); +unsigned long address,int map_size);static void update_domain (struct protection_domain *domain);#ifndef BUS_NOTIFY_UNBOUND_DRIVER
@@ -552,9 +552,9 @@ static int iommu_map_page(struct protection_domain *dom, }static void iommu_unmap_page (struct protection_domain *dom, -unsigned long bus_addr) +unsigned long bus_addr,int map_size) { -u64 *pte =fetch_pte (dom, bus_addr); +u64 *pte =fetch_pte (dom, bus_addr, map_size);if (pte) *pte =0 ;
@@ -668,7 +668,7 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,* there is one, it returns the pointer to it. */ static u64 *fetch_pte (struct protection_domain *domain, -unsigned long address) +unsigned long address,int map_size) {int level;u64 *pte;
@@ -676,7 +676,7 @@ static u64 *fetch_pte(struct protection_domain *domain, level = domain->mode -1 ; pte = &domain->pt_root[PM_LEVEL_INDEX (level, address)]; -while (level >0 ) { +while (level > map_size) {if (!IOMMU_PTE_PRESENT (*pte))return NULL ;
@@ -684,6 +684,11 @@ static u64 *fetch_pte(struct protection_domain *domain, pte =IOMMU_PTE_PAGE (*pte); pte = &pte[PM_LEVEL_INDEX (level, address)]; + +if ((PM_PTE_LEVEL (*pte) ==0 ) && level != map_size) { + pte =NULL ; +break ; + } }return pte;
@@ -757,7 +762,7 @@ static int alloc_new_range(struct amd_iommu *iommu,for (i = dma_dom->aperture[index]->offset; i < dma_dom->aperture_size; i +=PAGE_SIZE ) { -u64 *pte =fetch_pte (&dma_dom->domain, i); +u64 *pte =fetch_pte (&dma_dom->domain, i, PM_MAP_4k);if (!pte || !IOMMU_PTE_PRESENT (*pte))continue ;
@@ -2192,7 +2197,7 @@ static void amd_iommu_unmap_range(struct iommu_domain *dom, iova &=PAGE_MASK ;for (i =0 ; i < npages; ++i) { -iommu_unmap_page (domain, iova); +iommu_unmap_page (domain, iova, PM_MAP_4k); iova +=PAGE_SIZE ; }
@@ -2207,7 +2212,7 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,phys_addr_t paddr;u64 *pte; - pte =fetch_pte (domain, iova); + pte =fetch_pte (domain, iova, PM_MAP_4k);if (!pte || !IOMMU_PTE_PRESENT (*pte))return 0 ;
Signature · 1 line
1.6.3.3
PATCH 14/14 x86/amd-iommu: Change iommu_map_page to support multiple page sizes
In reply to Joerg RoedelThis patch adds a map_size parameter to the iommu_map_page function which makes it generic enough to handle multiple page sizes. This also requires a change to alloc_pte which is also done in this patch.
--- arch/x86/include/asm/amd_iommu_types.h | 6 ++++++ arch/x86/kernel/amd_iommu.c | 31 ++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 11 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +6 -0
@@ -160,6 +160,12 @@ IOMMU_PTE_P | IOMMU_PTE_IR | IOMMU_PTE_IW)#define PM_PTE_LEVEL (pte) (((pte) >> 9) & 0x7ULL) +#define PM_MAP_4k 0 +#define PM_ADDR_MASK 0x000ffffffffff000ULL +#define PM_MAP_MASK (lvl) (PM_ADDR_MASK & \ + (~((1ULL << (12 + ((lvl) * 9))) - 1))) +#define PM_ALIGNED (lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr)) +#define IOMMU_PTE_P (1ULL << 0)#define IOMMU_PTE_TV (1ULL << 1)#define IOMMU_PTE_U (1ULL << 59)
arch/x86/kernel/amd_iommu.c +20 -11
@@ -56,8 +56,8 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, struct unity_map_entry *e);static struct dma_ops_domain *find_protection_domain (u16 devid);static u64 *alloc_pte (struct protection_domain *domain, -unsigned long address,u64 - **pte_page,gfp_t gfp); +unsigned long address,int end_lvl, +u64 **pte_page,gfp_t gfp);static void dma_ops_reserve_addresses (struct dma_ops_domain *dom,unsigned long start_page,unsigned int pages);
@@ -523,17 +523,21 @@ void amd_iommu_flush_all_devices(void)static int iommu_map_page (struct protection_domain *dom,unsigned long bus_addr,unsigned long phys_addr, -int prot) +int prot, +int map_size) {u64 __pte, *pte; bus_addr =PAGE_ALIGN (bus_addr); phys_addr =PAGE_ALIGN (phys_addr); +BUG_ON (!PM_ALIGNED (map_size, bus_addr)); +BUG_ON (!PM_ALIGNED (map_size, phys_addr)); +if (!(prot &IOMMU_PROT_MASK ))return -EINVAL ; - pte =alloc_pte (dom, bus_addr,NULL ,GFP_KERNEL ); + pte =alloc_pte (dom, bus_addr, map_size,NULL ,GFP_KERNEL );if (IOMMU_PTE_PRESENT (*pte))return -EBUSY ;
@@ -612,7 +616,8 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,for (addr = e->address_start; addr < e->address_end; addr +=PAGE_SIZE ) { - ret =iommu_map_page (&dma_dom->domain, addr, addr, e->prot); + ret =iommu_map_page (&dma_dom->domain, addr, addr, e->prot, + PM_MAP_4k);if (ret)return ret;/*
@@ -729,7 +734,7 @@ static int alloc_new_range(struct amd_iommu *iommu,u64 *pte, *pte_page;for (i =0 ; i < num_ptes; ++i) { - pte =alloc_pte (&dma_dom->domain, address, + pte =alloc_pte (&dma_dom->domain, address, PM_MAP_4k, &pte_page, gfp);if (!pte) goto out_free;
@@ -1356,7 +1361,10 @@ static bool increase_address_space(struct protection_domain *domain, }static u64 *alloc_pte (struct protection_domain *domain, -unsigned long address,u64 **pte_page,gfp_t gfp) +unsigned long address, +int end_lvl, +u64 **pte_page, +gfp_t gfp) {u64 *pte, *page;int level;
@@ -1367,7 +1375,7 @@ static u64 *alloc_pte(struct protection_domain *domain, level = domain->mode -1 ; pte = &domain->pt_root[PM_LEVEL_INDEX (level, address)]; -while (level >0 ) { +while (level > end_lvl) {if (!IOMMU_PTE_PRESENT (*pte)) { page = (u64 *)get_zeroed_page (gfp);if (!page)
@@ -1379,7 +1387,7 @@ static u64 *alloc_pte(struct protection_domain *domain, pte =IOMMU_PTE_PAGE (*pte); -if (pte_page && level ==0 ) +if (pte_page && level == end_lvl) *pte_page = pte; pte = &pte[PM_LEVEL_INDEX (level, address)];
@@ -1403,7 +1411,8 @@ static u64* dma_ops_get_pte(struct dma_ops_domain *dom, pte = aperture->pte_pages[APERTURE_PAGE_INDEX (address)];if (!pte) { - pte =alloc_pte (&dom->domain, address, &pte_page,GFP_ATOMIC ); + pte =alloc_pte (&dom->domain, address, PM_MAP_4k, &pte_page, +GFP_ATOMIC ); aperture->pte_pages[APERTURE_PAGE_INDEX (address)] = pte_page; }else pte +=PM_LEVEL_INDEX (0 , address);
@@ -2176,7 +2185,7 @@ static int amd_iommu_map_range(struct iommu_domain *dom, paddr &=PAGE_MASK ;for (i =0 ; i < npages; ++i) { - ret =iommu_map_page (domain, iova, paddr, prot); + ret =iommu_map_page (domain, iova, paddr, prot, PM_MAP_4k);if (ret)return ret;
Signature · 1 line
1.6.3.3