Clean up huge vmap and ioremap code
Subsystems (6)
v2 cover Clean up huge vmap and ioremap code
Hi all,
This is version two of the patches I previously posted here:
http://lkml.kernel.org/r/[email protected]
Changes since v1 include:
* Fixed increment of the physical address around the mapping loops
* Added Reviewed-by tags from Toshi
All feedback welcome,
Will
--->8
Will Deacon (5):
ioremap: Rework pXd_free_pYd_page() API
arm64: mmu: Drop pXd_present() checks from pXd_free_pYd_table()
x86: pgtable: Drop pXd_none() checks from pXd_free_pYd_table() lib/ioremap: Ensure phys_addr actually corresponds to a physical
address
lib/ioremap: Ensure break-before-make is used for huge p4d mappings
arch/arm64/mm/mmu.c | 13 +++--- arch/x86/mm/pgtable.c | 14 +++--- include/asm-generic/pgtable.h | 5 ++ lib/ioremap.c | 103 +++++++++++++++++++++++++++++------------- 4 files changed, 91 insertions(+), 44 deletions(-)
Signature · 1 line
2.1.4
PATCH v2 1/5 ioremap: Rework pXd_free_pYd_page() API
The recently merged API for ensuring break-before-make on page-table entries when installing huge mappings in the vmalloc/ioremap region is fairly counter-intuitive, resulting in the arch freeing functions (e.g. pmd_free_pte_page()) being called even on entries that aren't present. This resulted in a minor bug in the arm64 implementation, giving rise to spurious VM_WARN messages.
This patch moves the pXd_present() checks out into the core code, refactoring the callsites at the same time so that we avoid the complex conjunctions when determining whether or not we can put down a huge mapping.
--- lib/ioremap.c | 56 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 14 deletions(-)
lib/ioremap.c +42 -14
@@ -76,6 +76,25 @@ static int ioremap_pte_range(pmd_t *pmd, unsigned long addr,return 0 ; } +static int ioremap_try_huge_pmd (pmd_t *pmd,unsigned long addr, +unsigned long end,phys_addr_t phys_addr, +pgprot_t prot) +{ +if (!ioremap_pmd_enabled ()) +return 0 ; + +if ((end - addr) !=PMD_SIZE ) +return 0 ; + +if (!IS_ALIGNED (phys_addr,PMD_SIZE )) +return 0 ; + +if (pmd_present (*pmd) && !pmd_free_pte_page (pmd, addr)) +return 0 ; + +return pmd_set_huge (pmd, phys_addr, prot); +} +static inline int ioremap_pmd_range (pud_t *pud,unsigned long addr,unsigned long end,phys_addr_t phys_addr,pgprot_t prot) {
@@ -89,13 +108,8 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, do { next = pmd_addr_end(addr, end); - if (ioremap_pmd_enabled() && - ((next - addr) == PMD_SIZE) && - IS_ALIGNED(phys_addr + addr, PMD_SIZE) && - pmd_free_pte_page(pmd, addr)) { - if (pmd_set_huge(pmd, phys_addr + addr, prot)) - continue; - } + if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr + addr, prot)) + continue; if (ioremap_pte_range(pmd, addr, next, phys_addr + addr, prot)) return -ENOMEM;
@@ -103,6 +117,25 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,return 0 ; } +static int ioremap_try_huge_pud (pud_t *pud,unsigned long addr, +unsigned long end,phys_addr_t phys_addr, +pgprot_t prot) +{ +if (!ioremap_pud_enabled ()) +return 0 ; + +if ((end - addr) !=PUD_SIZE ) +return 0 ; + +if (!IS_ALIGNED (phys_addr,PUD_SIZE )) +return 0 ; + +if (pud_present (*pud) && !pud_free_pmd_page (pud, addr)) +return 0 ; + +return pud_set_huge (pud, phys_addr, prot); +} +static inline int ioremap_pud_range (p4d_t *p4d,unsigned long addr,unsigned long end,phys_addr_t phys_addr,pgprot_t prot) {
@@ -116,13 +149,8 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, do { next = pud_addr_end(addr, end); - if (ioremap_pud_enabled() && - ((next - addr) == PUD_SIZE) && - IS_ALIGNED(phys_addr + addr, PUD_SIZE) && - pud_free_pmd_page(pud, addr)) { - if (pud_set_huge(pud, phys_addr + addr, prot)) - continue; - } + if (ioremap_try_huge_pud(pud, addr, next, phys_addr + addr, prot)) + continue; if (ioremap_pmd_range(pud, addr, next, phys_addr + addr, prot)) return -ENOMEM;
Signature · 1 line
2.1.4
PATCH v2 2/5 arm64: mmu: Drop pXd_present() checks from pXd_free_pYd_table()
In reply to Will DeaconNow that the core code checks this for us, we don't need to do it in the backend.
--- arch/arm64/mm/mmu.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
arch/arm64/mm/mmu.c +2 -6
@@ -985,10 +985,8 @@ int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) pmd =READ_ONCE (*pmdp); -if (!pmd_present (pmd)) -return 1 ;if (!pmd_table (pmd)) { -VM_WARN_ON (!pmd_table (pmd)); +VM_WARN_ON (1 );return 1 ; }
@@ -1008,10 +1006,8 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr) pud =READ_ONCE (*pudp); -if (!pud_present (pud)) -return 1 ;if (!pud_table (pud)) { -VM_WARN_ON (!pud_table (pud)); +VM_WARN_ON (1 );return 1 ; }
Signature · 1 line
2.1.4
PATCH v2 3/5 x86: pgtable: Drop pXd_none() checks from pXd_free_pYd_table()
In reply to Will DeaconNow that the core code checks this for us, we don't need to do it in the backend.
--- arch/x86/mm/pgtable.c | 6 ------ 1 file changed, 6 deletions(-)
arch/x86/mm/pgtable.c +0 -6
@@ -796,9 +796,6 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr)pte_t *pte;int i; -if (pud_none (*pud)) -return 1 ; - pmd = (pmd_t *)pud_page_vaddr (*pud); pmd_sv = (pmd_t *)__get_free_page (GFP_KERNEL );if (!pmd_sv)
@@ -840,9 +837,6 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) {pte_t *pte; -if (pmd_none (*pmd)) -return 1 ; - pte = (pte_t *)pmd_page_vaddr (*pmd);pmd_clear (pmd);
Signature · 1 line
2.1.4
Re: [PATCH v2 3/5] x86: pgtable: Drop pXd_none() checks from pXd_free_pYd_table()
On Tue, 2 Oct 2018, Will Deacon wrote:
Subject prefix wants to be 'x86/pgtable:' please
Quoted 2 lines
Now that the core code checks this for us, we don't need to do it in the backend.
According to Documentation changelogs want to be written in imperative mood.
The core code has a check for pXd_none() already. Remove it in the architecture implementation.
Or something like that.
Quoted 7 lines
Cc Chintan Pandya <[email protected]>Cc Toshi Kani <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Michal Hocko <[email protected]>Cc Andrew Morton <[email protected]>Reviewed-by Toshi Kani <[email protected]>Signed-off-by Will Deacon <[email protected]>
Other than the nits above:
PATCH v2 4/5 lib/ioremap: Ensure phys_addr actually corresponds to a physical address
In reply to Will DeaconThe current ioremap() code uses a phys_addr variable at each level of page table, which is confusingly offset by subtracting the base virtual address being mapped so that adding the current virtual address back on when iterating through the page table entries gives back the corresponding physical address.
This is fairly confusing and results in all users of phys_addr having to add the current virtual address back on. Instead, this patch just updates phys_addr when iterating over the page table entries, ensuring that it's always up-to-date and doesn't require explicit offsetting.
--- lib/ioremap.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-)
lib/ioremap.c +12 -16
@@ -101,19 +101,18 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,pmd_t *pmd;unsigned long next; - phys_addr -= addr; pmd =pmd_alloc (&init_mm, pud, addr);if (!pmd)return -ENOMEM ;do { next =pmd_addr_end (addr, end); -if (ioremap_try_huge_pmd (pmd, addr, next, phys_addr + addr, prot)) +if (ioremap_try_huge_pmd (pmd, addr, next, phys_addr, prot))continue ; -if (ioremap_pte_range (pmd, addr, next, phys_addr + addr, prot)) +if (ioremap_pte_range (pmd, addr, next, phys_addr, prot))return -ENOMEM ; - }while (pmd++, addr = next, addr != end); + }while (pmd++, phys_addr += (next - addr), addr = next, addr != end);return 0 ; }
@@ -142,19 +141,18 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,pud_t *pud;unsigned long next; - phys_addr -= addr; pud =pud_alloc (&init_mm, p4d, addr);if (!pud)return -ENOMEM ;do { next =pud_addr_end (addr, end); -if (ioremap_try_huge_pud (pud, addr, next, phys_addr + addr, prot)) +if (ioremap_try_huge_pud (pud, addr, next, phys_addr, prot))continue ; -if (ioremap_pmd_range (pud, addr, next, phys_addr + addr, prot)) +if (ioremap_pmd_range (pud, addr, next, phys_addr, prot))return -ENOMEM ; - }while (pud++, addr = next, addr != end); + }while (pud++, phys_addr += (next - addr), addr = next, addr != end);return 0 ; }
@@ -164,7 +162,6 @@ static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,p4d_t *p4d;unsigned long next; - phys_addr -= addr; p4d =p4d_alloc (&init_mm, pgd, addr);if (!p4d)return -ENOMEM ;
@@ -173,14 +170,14 @@ static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr,if (ioremap_p4d_enabled () && ((next - addr) ==P4D_SIZE ) && -IS_ALIGNED (phys_addr + addr,P4D_SIZE )) { -if (p4d_set_huge (p4d, phys_addr + addr, prot)) +IS_ALIGNED (phys_addr,P4D_SIZE )) { +if (p4d_set_huge (p4d, phys_addr, prot))continue ; } -if (ioremap_pud_range (p4d, addr, next, phys_addr + addr, prot)) +if (ioremap_pud_range (p4d, addr, next, phys_addr, prot))return -ENOMEM ; - }while (p4d++, addr = next, addr != end); + }while (p4d++, phys_addr += (next - addr), addr = next, addr != end);return 0 ; }
@@ -196,14 +193,13 @@ int ioremap_page_range(unsigned long addr,BUG_ON (addr >= end); start = addr; - phys_addr -= addr; pgd =pgd_offset_k (addr);do { next =pgd_addr_end (addr, end); - err =ioremap_p4d_range (pgd, addr, next, phys_addr+addr, prot); + err =ioremap_p4d_range (pgd, addr, next, phys_addr, prot);if (err)break ; - }while (pgd++, addr = next, addr != end); + }while (pgd++, phys_addr += (next - addr), addr = next, addr != end);flush_cache_vmap (start, end);
Signature · 1 line
2.1.4
PATCH v2 5/5 lib/ioremap: Ensure break-before-make is used for huge p4d mappings
In reply to Will DeaconWhilst no architectures actually enable support for huge p4d mappings in the vmap area, the code that is implemented should be using break-before-make, as we do for pud and pmd huge entries.
--- arch/arm64/mm/mmu.c | 5 +++++ arch/x86/mm/pgtable.c | 8 ++++++++ include/asm-generic/pgtable.h | 5 +++++ lib/ioremap.c | 27 +++++++++++++++++++++------ 4 files changed, 39 insertions(+), 6 deletions(-)
arch/arm64/mm/mmu.c +5 -0
@@ -1024,3 +1024,8 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr)pmd_free (NULL , table);return 1 ; } + +int p4d_free_pud_page (p4d_t *p4d,unsigned long addr) +{ +return 0 ;/* Don't attempt a block mapping */ +}
arch/x86/mm/pgtable.c +8 -0
@@ -779,6 +779,14 @@ int pmd_clear_huge(pmd_t *pmd)return 0 ; } +/* +* Until we support 512GB pages, skip them in the vmap area. +*/ +int p4d_free_pud_page (p4d_t *p4d,unsigned long addr) +{ +return 0 ; +} +#ifdef CONFIG_X86_64 /** * pud_free_pmd_page - Clear pud entry and free pmd page.
include/asm-generic/pgtable.h +5 -0
@@ -1019,6 +1019,7 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot);int pmd_set_huge (pmd_t *pmd,phys_addr_t addr,pgprot_t prot);int pud_clear_huge (pud_t *pud);int pmd_clear_huge (pmd_t *pmd); +int p4d_free_pud_page (p4d_t *p4d,unsigned long addr);int pud_free_pmd_page (pud_t *pud,unsigned long addr);int pmd_free_pte_page (pmd_t *pmd,unsigned long addr);#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
@@ -1046,6 +1047,10 @@ static inline int pmd_clear_huge(pmd_t *pmd) {return 0 ; } +static inline int p4d_free_pud_page (p4d_t *p4d,unsigned long addr) +{ +return 0 ; +}static inline int pud_free_pmd_page (pud_t *pud,unsigned long addr) {return 0 ;
lib/ioremap.c +21 -6
@@ -156,6 +156,25 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,return 0 ; } +static int ioremap_try_huge_p4d (p4d_t *p4d,unsigned long addr, +unsigned long end,phys_addr_t phys_addr, +pgprot_t prot) +{ +if (!ioremap_p4d_enabled ()) +return 0 ; + +if ((end - addr) !=P4D_SIZE ) +return 0 ; + +if (!IS_ALIGNED (phys_addr,P4D_SIZE )) +return 0 ; + +if (p4d_present (*p4d) && !p4d_free_pud_page (p4d, addr)) +return 0 ; + +return p4d_set_huge (p4d, phys_addr, prot); +} +static inline int ioremap_p4d_range (pgd_t *pgd,unsigned long addr,unsigned long end,phys_addr_t phys_addr,pgprot_t prot) {
@@ -168,12 +187,8 @@ static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, do { next = p4d_addr_end(addr, end); - if (ioremap_p4d_enabled() && - ((next - addr) == P4D_SIZE) && - IS_ALIGNED(phys_addr, P4D_SIZE)) { - if (p4d_set_huge(p4d, phys_addr, prot)) - continue; - } + if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) + continue; if (ioremap_pud_range(p4d, addr, next, phys_addr, prot)) return -ENOMEM;
Signature · 1 line
2.1.4