fixes for data/bss linear alias unmap series
Subsystems (5)
v2 cover fixes for data/bss linear alias unmap series
From: Ard Biesheuvel <[email protected]>
Fixes for the data/bss linear alias unmap series:
- Fix KASAN related issue reported by Mark, by moving all KASAN page tables out of BSS [on arm64], not just the ones defined under arch/arm64
- Fix two issues spotted by Sashiko
- Fix the CKI reported WARN() splat on BBML2_NOABORT systems that try to split block mappings too early.
Changes since v1:
- Split .bss..pgtbl changes off into separate patch
- Take Catalin's suggestion on how to address MTE issue with the read-only zero page
- Improved commit logs
Ard Biesheuvel (5):
arm64: Rename page table BSS section to .bss..pgtbl
kasan: Move generic KASAN page tables out of BSS too
arm64: Avoid double evaluation of __ptep_get()
KVM: arm64: Omit tag sync on stage-2 mappings of the zero page
arm64: mm: Defer remap of linear alias of data/bss
arch/arm64/include/asm/linkage.h | 2 ++ arch/arm64/include/asm/mmu.h | 2 -- arch/arm64/include/asm/pgtable.h | 4 ---- arch/arm64/kernel/vmlinux.lds.S | 8 ++++---- arch/arm64/kvm/mmu.c | 5 +++++ arch/arm64/mm/fixmap.c | 6 +++--- arch/arm64/mm/kasan_init.c | 2 +- arch/arm64/mm/mmu.c | 20 +++++++++++++------- include/linux/linkage.h | 4 ++++ mm/kasan/init.c | 10 +++++----- 10 files changed, 37 insertions(+), 26 deletions(-)
base-commit: 63e0b6a5b6934d6a919d1c65ea185303200a1874
Signature · 1 line
2.54.0.1032.g2f8565e1d1-goog
PATCH v2 1/5 arm64: Rename page table BSS section to .bss..pgtbl
From: Ard Biesheuvel <[email protected]>
Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will notice the leading ".bss" and mark it as NOBITS by default (rather than PROGBITS, which would take up space in Image binary, forcing all of the preceding BSS to be emitted into the image as well). This supersedes the NOLOAD linker directive, which achieves the same thing, and can be therefore be dropped.
Also, rename .pgdir to .pgtbl to be more generic, as page tables of various levels will reside here.
--- arch/arm64/include/asm/linkage.h | 2 ++ arch/arm64/include/asm/mmu.h | 2 -- arch/arm64/kernel/vmlinux.lds.S | 8 ++++---- arch/arm64/mm/fixmap.c | 6 +++--- arch/arm64/mm/kasan_init.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-)
arch/arm64/include/asm/linkage.h +2 -0
@@ -43,4 +43,6 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \bti c ; +#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE) +#endif
arch/arm64/include/asm/mmu.h +0 -2
@@ -13,8 +13,6 @@#ifndef __ASSEMBLER__ -#define __pgtbl_bss __section(".pgdir.bss") __aligned(PAGE_SIZE) -#include <linux/refcount.h> #include <asm/cpufeature.h>
arch/arm64/kernel/vmlinux.lds.S +4 -4
@@ -352,11 +352,11 @@ SECTIONS BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; - /* fixmap BSS starts here - preceding data/BSS is omitted from the linear map */ - .pgdir.bss (NOLOAD) : ALIGN(PAGE_SIZE) { - *(.pgdir.bss) + /* page table BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgtbl : ALIGN(PAGE_SIZE) { + *(.bss..pgtbl) } - ASSERT(ADDR(.pgdir.bss) == __bss_stop, ".pgdir.bss must follow BSS") + ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS") . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .;
arch/arm64/mm/fixmap.c +3 -3
@@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1);#define BM_PTE_TABLE_IDX (addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __pgtbl_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __pgtbl_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __pgtbl_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl; +static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused;static inline pte_t *fixmap_pte (unsigned long addr) {
arch/arm64/mm/kasan_init.c +1 -1
@@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void)* shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __pgtbl_bss; + static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl;pgd_t *pgdp =pgd_offset_k (KASAN_SHADOW_START );set_pgd (pgdp,__pgd (__pa_symbol (tbl) |PGD_TYPE_TABLE ));
Signature · 1 line
2.54.0.1032.g2f8565e1d1-goog
Re: [PATCH v2 1/5] arm64: Rename page table BSS section to .bss..pgtbl
Quoted 11 lines · Ard Biesheuvel
From: Ard Biesheuvel <[email protected]>
Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will notice the leading ".bss" and mark it as NOBITS by default (rather than PROGBITS, which would take up space in Image binary, forcing all of the preceding BSS to be emitted into the image as well). This supersedes the NOLOAD linker directive, which achieves the same thing, and can be therefore be dropped.
Also, rename .pgdir to .pgtbl to be more generic, as page tables of various levels will reside here.
This addresses the boot failure, I'm still seeing KUnit failures due to an unrelated bug that was fixed post -rc1 but we get into actually trying to run KUnit tests which was the relevant thing:
Re: [PATCH v2 1/5] arm64: Rename page table BSS section to .bss..pgtbl
In reply to Ard BiesheuvelQuoted 9 lines · Ard Biesheuvel
From: Ard Biesheuvel <[email protected]>
Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will notice the leading ".bss" and mark it as NOBITS by default (rather than PROGBITS, which would take up space in Image binary, forcing all of the preceding BSS to be emitted into the image as well). This supersedes the NOLOAD linker directive, which achieves the same thing, and can be therefore be dropped.
Quoted 5 lines
Also, rename .pgdir to .pgtbl to be more generic, as page tables of various levels will reside here.
Signed-off-by Ard Biesheuvel <[email protected]>---
I met boot failure for i.MX8QXP by this patch
[ 0.823515] Unable to handle kernel paging request at virtual address ffff00000328f000 [ 0.831116] Mem abort info: [ 0.833886] ESR = 0x0000000096000147 [ 0.837622] EC = 0x25: DABT (current EL), IL = 32 bits [ 0.842923] SET = 0, FnV = 0 [ 0.845961] EA = 0, S1PTW = 0 [ 0.849088] FSC = 0x07: level 3 translation fault [ 0.853952] Data abort info: [ 0.856809] ISV = 0, ISS = 0x00000147, ISS2 = 0x00000000 [ 0.862296] CM = 1, WnR = 1, TnD = 0, TagAccess = 0 [ 0.867330] GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0 [ 0.872633] swapper pgtable: 4k pages, 48-bit VAs, pgdp=000000008211f000 [ 0.879321] [ffff00000328f000] pgd=0000000000000000, p4d=18000008bffff403, pud=18000008bfffe403, pmd=18000008bffea403, pte=00e800008328ff06 [ 0.891834] Internal error: Oops: 0000000096000147 [#1] SMP [ 0.897469] Modules linked in: [ 0.900514] CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 7.1.0-rc1-00016-g63e0b6a5b693 #834 PREEMPT [ 0.909978] Hardware name: Freescale i.MX8QXP MEK (DT) [ 0.915104] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.922053] pc : dcache_clean_inval_poc+0x24/0x48 [ 0.926742] lr : kvm_arm_init+0xa78/0x1638 [ 0.930828] sp : ffff80008318bd10 [ 0.934127] x29: ffff80008318bd50 x28: 0000000000000000 x27: ffff00000328f000 [ 0.941251] x26: 0000000000002000 x25: ffff80008219e000 x24: 0000000001002222 [ 0.948374] x23: 0000000000000030 x22: ffff800081e850c0 x21: ffff800082b790d0 [ 0.955498] x20: 0000000000000004 x19: ffff8000830a0000 x18: 0000000000000000 [ 0.962622] x17: ffff800082f938b8 x16: ffff800082b8b4e0 x15: ffff800082b8b4b8 [ 0.969746] x14: ffff80008308f0a0 x13: ffff800082b8b490 x12: ffff800082b8b530 [ 0.976869] x11: ffff800082b8b508 x10: ffff80008308f140 x9 : ffff80008308f118 [ 0.983993] x8 : ffff80008308f0f0 x7 : ffff80008308f0c8 x6 : ffff80008308f078 [ 0.991117] x5 : ffff80008308f050 x4 : ffff800082b8b468 x3 : 000000000000003f [ 0.998240] x2 : 0000000000000040 x1 : ffff000003291000 x0 : ffff00000328f000 [ 1.005367] Call trace: [ 1.007800] dcache_clean_inval_poc+0x24/0x48 (P) [ 1.012490] do_one_initcall+0x80/0x1c8 [ 1.016310] kernel_init_freeable+0x208/0x2f0 [ 1.020654] kernel_init+0x24/0x1e0 [ 1.024131] ret_from_fork+0x10/0x20 [ 1.027700] Code: 9ac32042 d1000443 8a230000 d503201f (d50b7e20) [ 1.033779] ---[ end trace 0000000000000000 ]--- [ 1.038428] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b [ 1.046026] SMP: stopping secondary CPUs [ 1.049943] Kernel Offset: disabled [ 1.053408] CPU features: 0x00000000,00000008,00040021,0400421b [ 1.059316] Memory Limit: none [ 1.062359] ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b ]---
Any idea?
Frank
Quoted 84 lines
arch/arm64/include/asm/linkage.h | 2 ++ arch/arm64/include/asm/mmu.h | 2 -- arch/arm64/kernel/vmlinux.lds.S | 8 ++++---- arch/arm64/mm/fixmap.c | 6 +++--- arch/arm64/mm/kasan_init.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-)arch/arm64/include/asm/linkage.h +2
-0@@ -43,4 +43,6 @@ SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \bti c ; +#define __bss_pgtbl __section(".bss..pgtbl") __aligned(PAGE_SIZE) +#endif arch/arm64/include/asm/mmu.h +0
-2@@ -13,8 +13,6 @@#ifndef __ASSEMBLER__ -#define __pgtbl_bss __section(".pgdir.bss") __aligned(PAGE_SIZE) -#include <linux/refcount.h> #include <asm/cpufeature.h> arch/arm64/kernel/vmlinux.lds.S +4
-4@@ -352,11 +352,11 @@ SECTIONS BSS_SECTION(SBSS_ALIGN, 0, PAGE_SIZE) __pi___bss_start = __bss_start; - /* fixmap BSS starts here - preceding data/BSS is omitted from the linear map */ - .pgdir.bss (NOLOAD) : ALIGN(PAGE_SIZE) { - *(.pgdir.bss) + /* page table BSS starts here - preceding data/BSS is omitted from the linear map */ + .pgtbl : ALIGN(PAGE_SIZE) { + *(.bss..pgtbl) } - ASSERT(ADDR(.pgdir.bss) == __bss_stop, ".pgdir.bss must follow BSS") + ASSERT(ADDR(.pgtbl) == __bss_stop, ".pgtbl must follow BSS") . = ALIGN(PAGE_SIZE); __pi_init_pg_dir = .;arch/arm64/mm/fixmap.c +3
-3@@ -31,9 +31,9 @@ static_assert(NR_BM_PMD_TABLES == 1);#define BM_PTE_TABLE_IDX (addr) __BM_TABLE_IDX(addr, PMD_SHIFT) -static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __pgtbl_bss; -static pmd_t bm_pmd[PTRS_PER_PMD] __pgtbl_bss __maybe_unused; -static pud_t bm_pud[PTRS_PER_PUD] __pgtbl_bss __maybe_unused; +static pte_t bm_pte[NR_BM_PTE_TABLES][PTRS_PER_PTE] __bss_pgtbl; +static pmd_t bm_pmd[PTRS_PER_PMD] __bss_pgtbl __maybe_unused; +static pud_t bm_pud[PTRS_PER_PUD] __bss_pgtbl __maybe_unused;static inline pte_t *fixmap_pte (unsigned long addr) {arch/arm64/mm/kasan_init.c +1
-2@@ -214,7 +214,7 @@ asmlinkage void __init kasan_early_init(void)* shadow pud_t[]/p4d_t[], which could end up getting corrupted * when the linear region is mapped. */ - static pte_t tbl[PTRS_PER_PTE] __pgtbl_bss; + static pte_t tbl[PTRS_PER_PTE] __bss_pgtbl;pgd_t *pgdp =pgd_offset_k (KASAN_SHADOW_START );set_pgd (pgdp,__pgd (__pa_symbol (tbl) |PGD_TYPE_TABLE )); --2.54.0.1032.g2f8565e1d1-goog
Re: [PATCH v2 1/5] arm64: Rename page table BSS section to .bss..pgtbl
Quoted 80 lines · Frank Li
Quoted 9 lines · Ard Biesheuvel
From: Ard Biesheuvel <[email protected]>
Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will notice the leading ".bss" and mark it as NOBITS by default (rather than PROGBITS, which would take up space in Image binary, forcing all of the preceding BSS to be emitted into the image as well). This supersedes the NOLOAD linker directive, which achieves the same thing, and can be therefore be dropped.
Quoted 5 lines
Also, rename .pgdir to .pgtbl to be more generic, as page tables of various levels will reside here.
Signed-off-by Ard Biesheuvel <[email protected]>---
I met boot failure for i.MX8QXP by this patch
[ 0.823515] Unable to handle kernel paging request at virtual address ffff00000328f000 [ 0.831116] Mem abort info: [ 0.833886] ESR = 0x0000000096000147 [ 0.837622] EC = 0x25: DABT (current EL), IL = 32 bits [ 0.842923] SET = 0, FnV = 0 [ 0.845961] EA = 0, S1PTW = 0 [ 0.849088] FSC = 0x07: level 3 translation fault [ 0.853952] Data abort info: [ 0.856809] ISV = 0, ISS = 0x00000147, ISS2 = 0x00000000 [ 0.862296] CM = 1, WnR = 1, TnD = 0, TagAccess = 0 [ 0.867330] GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0 [ 0.872633] swapper pgtable: 4k pages, 48-bit VAs, pgdp=000000008211f000 [ 0.879321] [ffff00000328f000] pgd=0000000000000000, p4d=18000008bffff403, pud=18000008bfffe403, pmd=18000008bffea403, pte=00e800008328ff06 [ 0.891834] Internal error: Oops: 0000000096000147 [#1] SMP [ 0.897469] Modules linked in: [ 0.900514] CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 7.1.0-rc1-00016-g63e0b6a5b693 #834 PREEMPT [ 0.909978] Hardware name: Freescale i.MX8QXP MEK (DT) [ 0.915104] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.922053] pc : dcache_clean_inval_poc+0x24/0x48 [ 0.926742] lr : kvm_arm_init+0xa78/0x1638 [ 0.930828] sp : ffff80008318bd10 [ 0.934127] x29: ffff80008318bd50 x28: 0000000000000000 x27: ffff00000328f000 [ 0.941251] x26: 0000000000002000 x25: ffff80008219e000 x24: 0000000001002222 [ 0.948374] x23: 0000000000000030 x22: ffff800081e850c0 x21: ffff800082b790d0 [ 0.955498] x20: 0000000000000004 x19: ffff8000830a0000 x18: 0000000000000000 [ 0.962622] x17: ffff800082f938b8 x16: ffff800082b8b4e0 x15: ffff800082b8b4b8 [ 0.969746] x14: ffff80008308f0a0 x13: ffff800082b8b490 x12: ffff800082b8b530 [ 0.976869] x11: ffff800082b8b508 x10: ffff80008308f140 x9 : ffff80008308f118 [ 0.983993] x8 : ffff80008308f0f0 x7 : ffff80008308f0c8 x6 : ffff80008308f078 [ 0.991117] x5 : ffff80008308f050 x4 : ffff800082b8b468 x3 : 000000000000003f [ 0.998240] x2 : 0000000000000040 x1 : ffff000003291000 x0 : ffff00000328f000 [ 1.005367] Call trace: [ 1.007800] dcache_clean_inval_poc+0x24/0x48 (P) [ 1.012490] do_one_initcall+0x80/0x1c8 [ 1.016310] kernel_init_freeable+0x208/0x2f0 [ 1.020654] kernel_init+0x24/0x1e0 [ 1.024131] ret_from_fork+0x10/0x20 [ 1.027700] Code: 9ac32042 d1000443 8a230000 d503201f (d50b7e20) [ 1.033779] ---[ end trace 0000000000000000 ]--- [ 1.038428] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b [ 1.046026] SMP: stopping secondary CPUs [ 1.049943] Kernel Offset: disabled [ 1.053408] CPU features: 0x00000000,00000008,00040021,0400421b [ 1.059316] Memory Limit: none [ 1.062359] ---[ end Kernel panic - not syncing: Attempted to kill
Quoted 5 lines
init! exitcode=0x0000000b ]---
Any idea?
Which tree is this based on?
Re: [PATCH v2 1/5] arm64: Rename page table BSS section to .bss..pgtbl
Quoted 86 lines · Ard Biesheuvel
Quoted 83 lines · Frank Li
Quoted 9 lines · Ard Biesheuvel
From: Ard Biesheuvel <[email protected]>
Rename the .pgdir.bss section to .bss..pgtbl so that the compiler will notice the leading ".bss" and mark it as NOBITS by default (rather than PROGBITS, which would take up space in Image binary, forcing all of the preceding BSS to be emitted into the image as well). This supersedes the NOLOAD linker directive, which achieves the same thing, and can be therefore be dropped.
Quoted 5 lines
Also, rename .pgdir to .pgtbl to be more generic, as page tables of various levels will reside here.
Signed-off-by Ard Biesheuvel <[email protected]>---
I met boot failure for i.MX8QXP by this patch
[ 0.823515] Unable to handle kernel paging request at virtual address ffff00000328f000 [ 0.831116] Mem abort info: [ 0.833886] ESR = 0x0000000096000147 [ 0.837622] EC = 0x25: DABT (current EL), IL = 32 bits [ 0.842923] SET = 0, FnV = 0 [ 0.845961] EA = 0, S1PTW = 0 [ 0.849088] FSC = 0x07: level 3 translation fault [ 0.853952] Data abort info: [ 0.856809] ISV = 0, ISS = 0x00000147, ISS2 = 0x00000000 [ 0.862296] CM = 1, WnR = 1, TnD = 0, TagAccess = 0 [ 0.867330] GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0 [ 0.872633] swapper pgtable: 4k pages, 48-bit VAs, pgdp=000000008211f000 [ 0.879321] [ffff00000328f000] pgd=0000000000000000, p4d=18000008bffff403, pud=18000008bfffe403, pmd=18000008bffea403, pte=00e800008328ff06 [ 0.891834] Internal error: Oops: 0000000096000147 [#1] SMP [ 0.897469] Modules linked in: [ 0.900514] CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 7.1.0-rc1-00016-g63e0b6a5b693 #834 PREEMPT [ 0.909978] Hardware name: Freescale i.MX8QXP MEK (DT) [ 0.915104] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 0.922053] pc : dcache_clean_inval_poc+0x24/0x48 [ 0.926742] lr : kvm_arm_init+0xa78/0x1638 [ 0.930828] sp : ffff80008318bd10 [ 0.934127] x29: ffff80008318bd50 x28: 0000000000000000 x27: ffff00000328f000 [ 0.941251] x26: 0000000000002000 x25: ffff80008219e000 x24: 0000000001002222 [ 0.948374] x23: 0000000000000030 x22: ffff800081e850c0 x21: ffff800082b790d0 [ 0.955498] x20: 0000000000000004 x19: ffff8000830a0000 x18: 0000000000000000 [ 0.962622] x17: ffff800082f938b8 x16: ffff800082b8b4e0 x15: ffff800082b8b4b8 [ 0.969746] x14: ffff80008308f0a0 x13: ffff800082b8b490 x12: ffff800082b8b530 [ 0.976869] x11: ffff800082b8b508 x10: ffff80008308f140 x9 : ffff80008308f118 [ 0.983993] x8 : ffff80008308f0f0 x7 : ffff80008308f0c8 x6 : ffff80008308f078 [ 0.991117] x5 : ffff80008308f050 x4 : ffff800082b8b468 x3 : 000000000000003f [ 0.998240] x2 : 0000000000000040 x1 : ffff000003291000 x0 : ffff00000328f000 [ 1.005367] Call trace: [ 1.007800] dcache_clean_inval_poc+0x24/0x48 (P) [ 1.012490] do_one_initcall+0x80/0x1c8 [ 1.016310] kernel_init_freeable+0x208/0x2f0 [ 1.020654] kernel_init+0x24/0x1e0 [ 1.024131] ret_from_fork+0x10/0x20 [ 1.027700] Code: 9ac32042 d1000443 8a230000 d503201f (d50b7e20) [ 1.033779] ---[ end trace 0000000000000000 ]--- [ 1.038428] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b [ 1.046026] SMP: stopping secondary CPUs [ 1.049943] Kernel Offset: disabled [ 1.053408] CPU features: 0x00000000,00000008,00040021,0400421b [ 1.059316] Memory Limit: none [ 1.062359] ---[ end Kernel panic - not syncing: Attempted to killinit! exitcode=0x0000000b ]---
Quoted 4 lines
Quoted 2 lines
Any idea?
Which tree is this based on?
Sorry, after rebase to 20260615 linux-next, problem disappear. Problem appear at next-20260608. Suppose some patch fixed this problem recently.
Frank
PATCH v2 2/5 kasan: Move generic KASAN page tables out of BSS too
In reply to Ard BiesheuvelFrom: Ard Biesheuvel <[email protected]>
Make sure that all KASAN page tables are emitted into the .pgtbl section (provided that the arch has one - otherwise, fall back to page aligned BSS)
This is needed because BSS itself is no longer accessible via the linear map on arm64.
--- include/linux/linkage.h | 4 ++++ mm/kasan/init.c | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-)
include/linux/linkage.h +4 -0
@@ -39,6 +39,10 @@#define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE)#define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE) +#ifndef __bss_pgtbl +#define __bss_pgtbl __page_aligned_bss +#endif +/* * For assembly routines. *
mm/kasan/init.c +5 -5
@@ -26,10 +26,10 @@* - Latter it reused it as zero shadow to cover large ranges of memory * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...). */ -unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss; +unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl;#if CONFIG_PGTABLE_LEVELS >4 -p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss; +p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl;static inline bool kasan_p4d_table (pgd_t pgd) {return pgd_page (pgd) ==virt_to_page (lm_alias (kasan_early_shadow_p4d));
@@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd) }#endif #if CONFIG_PGTABLE_LEVELS >3 -pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss; +pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl;static inline bool kasan_pud_table (p4d_t p4d) {return p4d_page (p4d) ==virt_to_page (lm_alias (kasan_early_shadow_pud));
@@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d) }#endif #if CONFIG_PGTABLE_LEVELS >2 -pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss; +pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl;static inline bool kasan_pmd_table (pud_t pud) {return pud_page (pud) ==virt_to_page (lm_alias (kasan_early_shadow_pmd));
@@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud) }#endif pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS] - __page_aligned_bss; + __bss_pgtbl;static inline bool kasan_pte_table (pmd_t pmd) {
Signature · 1 line
2.54.0.1032.g2f8565e1d1-goog
PATCH v2 3/5 arm64: Avoid double evaluation of __ptep_get()
In reply to Ard BiesheuvelFrom: Ard Biesheuvel <[email protected]>
Sashiko warns that the new pte_valid_noncont() macro is used in a manner where the argument (which performs a READ_ONCE() of the descriptor) is evaluated twice.
Drop the macro that we just added, and move the check into the newly added users.
--- arch/arm64/include/asm/pgtable.h | 4 ---- arch/arm64/mm/mmu.c | 14 ++++++++++---- 2 files changed, 10 insertions(+), 8 deletions(-)
arch/arm64/include/asm/pgtable.h +0 -4
@@ -181,10 +181,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys)* Returns true if the pte is valid and has the contiguous bit set. */ #define pte_valid_cont (pte) (pte_valid(pte) && pte_cont(pte)) -/* -* Returns true if the pte is valid and has the contiguous bit cleared. -*/ -#define pte_valid_noncont (pte) (pte_valid(pte) && !pte_cont(pte))/* * Could the pte be present in the TLB? We must check mm_tlb_flush_pending * so that we don't erroneously return false for pages that have been
arch/arm64/mm/mmu.c +10 -4
@@ -186,9 +186,12 @@ static void init_pte(pte_t *ptep, unsigned long addr, unsigned long end,static bool pte_range_has_valid_noncont (pte_t *ptep) { -for (int i =0 ; i <CONT_PTES ; i++) -if (pte_valid_noncont (__ptep_get (&ptep[i]))) +for (int i =0 ; i <CONT_PTES ; i++) { +pte_t pte =__ptep_get (&ptep[i]); + +if (pte_valid (pte) && !pte_cont (pte))return true; + }return false; }
@@ -291,9 +294,12 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,static bool pmd_range_has_valid_noncont (pmd_t *pmdp) { -for (int i =0 ; i <CONT_PMDS ; i++) -if (pte_valid_noncont (pmd_pte (READ_ONCE (pmdp[i])))) +for (int i =0 ; i <CONT_PMDS ; i++) { +pte_t pte =pmd_pte (READ_ONCE (pmdp[i])); + +if (pte_valid (pte) && !pte_cont (pte))return true; + }return false; }
Signature · 1 line
2.54.0.1032.g2f8565e1d1-goog
PATCH v2 4/5 KVM: arm64: Omit tag sync on stage-2 mappings of the zero page
In reply to Ard BiesheuvelFrom: Ard Biesheuvel <[email protected]>
Commit
f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged")
removed the PG_mte_tagged flag from the zero page, but missed a KVM code path that may set this flag on the zero page when it is used in a stage-2 CoW mapping of anonymous memory.
So disregard the zero page explicitly in sanitise_mte_tags().
--- arch/arm64/kvm/mmu.c | 5 +++++ 1 file changed, 5 insertions(+)
arch/arm64/kvm/mmu.c +5 -0
@@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,if (!kvm_has_mte (kvm))return ; +if (is_zero_pfn (pfn)) { +WARN_ON_ONCE (nr_pages !=1 ); +return ; + } +if (folio_test_hugetlb (folio)) {/* Hugetlb has MTE flags set on head page only */ if (folio_try_hugetlb_mte_tagging (folio)) {
Signature · 1 line
2.54.0.1032.g2f8565e1d1-goog
Re: [PATCH v2 4/5] KVM: arm64: Omit tag sync on stage-2 mappings of the zero page
Quoted 10 lines · Ard Biesheuvel
From: Ard Biesheuvel <[email protected]>
Commit
f620d66af316("arm64: mte: Do not flag the zero page asPG_mte_tagged")removed the
PG_mte_taggedflag from the zero page, but missed a KVM code path that may set this flag on the zero page when it is used in a stage-2 CoW mapping of anonymous memory.
Quoted 6 lines
So disregard the zero page explicitly in
sanitise_mte_tags().Fixes f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged")Cc <[email protected]> # 5.10.xSuggested-by Catalin Marinas <[email protected]>Signed-off-by Ard Biesheuvel <[email protected]>
Re: [PATCH v2 4/5] KVM: arm64: Omit tag sync on stage-2 mappings of the zero page
In reply to Ard BiesheuvelQuoted 27 lines · Ard Biesheuvel
From: Ard Biesheuvel <[email protected]>
Commit
f620d66af316("arm64: mte: Do not flag the zero page asPG_mte_tagged")removed the
PG_mte_taggedflag from the zero page, but missed a KVM code path that may set this flag on the zero page when it is used in a stage-2 CoW mapping of anonymous memory.So disregard the zero page explicitly in
sanitise_mte_tags().Fixes f620d66af316 ("arm64: mte: Do not flag the zero page as PG_mte_tagged")Cc <[email protected]> # 5.10.xSuggested-by Catalin Marinas <[email protected]>Signed-off-by Ard Biesheuvel <[email protected]>--- arch/arm64/kvm/mmu.c | 5 +++++ 1 file changed, 5 insertions(+)arch/arm64/kvm/mmu.c +0
-0@@ -1479,6 +1479,11 @@ static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn,if (!kvm_has_mte (kvm))
arch/arm64/kvm/mmu.c static void sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, · line 1489
Quoted 10 lines
arch/arm64/kvm/mmu.c +5
-0return ; +if (is_zero_pfn (pfn)) { +WARN_ON_ONCE (nr_pages !=1 ); +return ; + } +if (folio_test_hugetlb (folio)) {/* Hugetlb has MTE flags set on head page only */ if (folio_try_hugetlb_mte_tagging (folio)) {
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
PATCH v2 5/5 arm64: mm: Defer remap of linear alias of data/bss
In reply to Ard BiesheuvelFrom: Ard Biesheuvel <[email protected]>
Marking the linear alias of data/bss invalid involves calling set_memory_valid(), which calls split_kernel_leaf_mapping() under the hood.
On BBML2_NOABORT capable systems, this may result in the need to allocate page tables at a time when the generic memory allocation APIs are not yet available, resulting in a splat like
WARNING: arch/arm64/mm/mmu.c:821 at split_kernel_leaf_mapping+0x15c/0x170, CPU#0: swapper/0
Modules linked in:
CPU: 0 UID: 0 PID: 0 Comm: swapper Not tainted 7.1.0-rc6 #1 PREEMPT(undef)
pstate: a04000c9 (NzCv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
pc : split_kernel_leaf_mapping+0x15c/0x170
lr : update_range_prot+0x40/0x128
sp : ffffc99ad3863c80
...
Call trace:
split_kernel_leaf_mapping+0x15c/0x170 (P)
update_range_prot+0x40/0x128
set_memory_valid+0x94/0xe0
mark_linear_data_alias_valid+0x54/0x68
map_mem+0x1fc/0x240
paging_init+0x48/0x210
setup_arch+0x274/0x338
start_kernel+0x98/0x538
__primary_switched+0x88/0x98as reported by CKI automated testing.
So defer the boot-time call to mark_linear_data_alias_valid() to a later time when page allocations can be made normally.
--- arch/arm64/mm/mmu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
arch/arm64/mm/mmu.c +3 -3
@@ -1095,6 +1095,9 @@ void __init mark_linear_text_alias_ro(void) (unsigned long )__init_begin - (unsigned long )_text, PAGE_KERNEL_RO); +/* Map the kernel data/bss as invalid in the linear map */ +mark_linear_data_alias_valid (false); +/* * Register a PM notifier to remap the linear alias of data/bss as * valid read-only before hibernation. This is needed because the
@@ -1237,9 +1240,6 @@ static void __init map_mem(void)__map_memblock (start, end,pgprot_tagged (PAGE_KERNEL ), flags); } - -/* Map the kernel data/bss as invalid in the linear map */ -mark_linear_data_alias_valid (false); }void mark_rodata_ro (void )
Signature · 1 line
2.54.0.1032.g2f8565e1d1-goog
Quoted 9 lines · Ard Biesheuvel
Fixes for the data/bss linear alias unmap series:
- Fix KASAN related issue reported by Mark, by moving all KASAN page tables out of BSS [on arm64], not just the ones defined under
arch/arm64- Fix two issues spotted by Sashiko
[...]
Applied to arm64 (for-next/mm), thanks!
[1/5] arm64: Rename page table BSS section to .bss..pgtbl
https://git.kernel.org/arm64/c/9c401fa7398f [2/5] kasan: Move generic KASAN page tables out of BSS too https://git.kernel.org/arm64/c/9f7f685758c6 [3/5] arm64: Avoid double evaluation of __ptep_get() https://git.kernel.org/arm64/c/568def8e87fc [4/5] KVM: arm64: Omit tag sync on stage-2 mappings of the zero page https://git.kernel.org/arm64/c/2986a6257405 [5/5] arm64: mm: Defer remap of linear alias of data/bss https://git.kernel.org/arm64/c/53205d56212c
Cheers,
Signature · 5 lines
Will https://fixes.arm64.dev https://next.arm64.dev https://will.arm64.dev