x86/kasan: Populate shadow for read-only IDT mapping
cover x86/kasan: Populate shadow for read-only IDT mapping
Fix a regression introduced by mapping shadows for the per-cpu portions of the CPU entry area on-demand. The read-only IDT mapping is also shoved into the CPU entry area, but since it's shared, no CPU creates a shadow for it. KVM on Intel does an IDT lookup in software when handling host IRQs that arrived in the guest, which results in KASAN dereferencing an unmapped shadow.
The first two patches are cleanups to make the fix (and code in general) less ugly.
Side topic, KASAN should really decide whether it wants to use "void *" or "unsigned long", e.g. kasan_populate_shadow() takes "unsigned long" but kasan_populate_early_shadow() takes "void *". And the amount of casting throughout the code is bonkers.
Sean Christopherson (3):
x86/kasan: Rename local CPU_ENTRY_AREA variables to shorten names x86/kasan: Add helpers to align shadow addresses up and down x86/kasan: Populate shadow for shared chunk of the CPU entry area
arch/x86/mm/kasan_init_64.c | 50 ++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 18 deletions(-)
base-commit: 3301badde43dee7c2a013fbd6479c258366519da
Signature · 1 line
2.38.1.431.g37b22c650d-goog
PATCH 1/3 x86/kasan: Rename local CPU_ENTRY_AREA variables to shorten names
Rename the CPU entry area variables in kasan_init() to shorten their names, a future fix will reference the beginning of the per-CPU portion of the CPU entry area, and shadow_cpu_entry_per_cpu_begin is a bit much.
No functional change intended.
--- arch/x86/mm/kasan_init_64.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-)
arch/x86/mm/kasan_init_64.c +11 -11
@@ -331,7 +331,7 @@ void __init kasan_populate_shadow_for_vaddr(void *va, size_t size, int nid)void __init kasan_init (void ) {int i; -void *shadow_cpu_entry_begin, *shadow_cpu_entry_end; +void *shadow_cea_begin, *shadow_cea_end;memcpy (early_top_pgt, init_top_pgt,sizeof (early_top_pgt));
@@ -372,16 +372,16 @@ void __init kasan_init(void)map_range (&pfn_mapped[i]); } - shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE ; - shadow_cpu_entry_begin =kasan_mem_to_shadow (shadow_cpu_entry_begin); - shadow_cpu_entry_begin = (void *)round_down ( - (unsigned long )shadow_cpu_entry_begin,PAGE_SIZE ); + shadow_cea_begin = (void *)CPU_ENTRY_AREA_BASE ; + shadow_cea_begin =kasan_mem_to_shadow (shadow_cea_begin); + shadow_cea_begin = (void *)round_down ( + (unsigned long )shadow_cea_begin,PAGE_SIZE ); - shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE + + shadow_cea_end = (void *)(CPU_ENTRY_AREA_BASE +CPU_ENTRY_AREA_MAP_SIZE ); - shadow_cpu_entry_end =kasan_mem_to_shadow (shadow_cpu_entry_end); - shadow_cpu_entry_end = (void *)round_up ( - (unsigned long )shadow_cpu_entry_end,PAGE_SIZE ); + shadow_cea_end =kasan_mem_to_shadow (shadow_cea_end); + shadow_cea_end = (void *)round_up ( + (unsigned long )shadow_cea_end,PAGE_SIZE ); kasan_populate_early_shadow( kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
@@ -403,9 +403,9 @@ void __init kasan_init(void)kasan_populate_early_shadow (kasan_mem_to_shadow ((void *)VMALLOC_END +1 ), - shadow_cpu_entry_begin); + shadow_cea_begin); -kasan_populate_early_shadow (shadow_cpu_entry_end, +kasan_populate_early_shadow (shadow_cea_end,kasan_mem_to_shadow ((void *)__START_KERNEL_map)); kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext),
Signature · 1 line
2.38.1.431.g37b22c650d-goog
PATCH 2/3 x86/kasan: Add helpers to align shadow addresses up and down
In reply to Sean ChristophersonAdd helpers to dedup code for aligning shadow address up/down to page boundaries when translating an address to its shadow.
No functional change intended.
--- arch/x86/mm/kasan_init_64.c | 40 ++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-)
arch/x86/mm/kasan_init_64.c +22 -18
@@ -316,22 +316,33 @@ void __init kasan_early_init(void)kasan_map_early_shadow (init_top_pgt); } +static unsigned long kasan_mem_to_shadow_align_down (unsigned long va) +{ +unsigned long shadow = (unsigned long )kasan_mem_to_shadow ((void *)va); + +return round_down (shadow,PAGE_SIZE ); +} + +static unsigned long kasan_mem_to_shadow_align_up (unsigned long va) +{ +unsigned long shadow = (unsigned long )kasan_mem_to_shadow ((void *)va); + +return round_up (shadow,PAGE_SIZE ); +} +void __init kasan_populate_shadow_for_vaddr (void *va,size_t size,int nid) {unsigned long shadow_start, shadow_end; - shadow_start = (unsigned long )kasan_mem_to_shadow (va); - shadow_start =round_down (shadow_start,PAGE_SIZE ); - shadow_end = (unsigned long )kasan_mem_to_shadow (va + size); - shadow_end =round_up (shadow_end,PAGE_SIZE ); - + shadow_start =kasan_mem_to_shadow_align_down ((unsigned long )va); + shadow_end =kasan_mem_to_shadow_align_up ((unsigned long )va + size);kasan_populate_shadow (shadow_start, shadow_end, nid); }void __init kasan_init (void ) { +unsigned long shadow_cea_begin, shadow_cea_end;int i; -void *shadow_cea_begin, *shadow_cea_end;memcpy (early_top_pgt, init_top_pgt,sizeof (early_top_pgt));
@@ -372,16 +383,9 @@ void __init kasan_init(void)map_range (&pfn_mapped[i]); } - shadow_cea_begin = (void *)CPU_ENTRY_AREA_BASE ; - shadow_cea_begin =kasan_mem_to_shadow (shadow_cea_begin); - shadow_cea_begin = (void *)round_down ( - (unsigned long )shadow_cea_begin,PAGE_SIZE ); - - shadow_cea_end = (void *)(CPU_ENTRY_AREA_BASE + -CPU_ENTRY_AREA_MAP_SIZE ); - shadow_cea_end =kasan_mem_to_shadow (shadow_cea_end); - shadow_cea_end = (void *)round_up ( - (unsigned long )shadow_cea_end,PAGE_SIZE ); + shadow_cea_begin =kasan_mem_to_shadow_align_down (CPU_ENTRY_AREA_BASE ); + shadow_cea_end =kasan_mem_to_shadow_align_up (CPU_ENTRY_AREA_BASE + +CPU_ENTRY_AREA_MAP_SIZE ); kasan_populate_early_shadow( kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
@@ -403,9 +407,9 @@ void __init kasan_init(void)kasan_populate_early_shadow (kasan_mem_to_shadow ((void *)VMALLOC_END +1 ), - shadow_cea_begin); + (void *)shadow_cea_begin); -kasan_populate_early_shadow (shadow_cea_end, +kasan_populate_early_shadow ((void *)shadow_cea_end,kasan_mem_to_shadow ((void *)__START_KERNEL_map)); kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext),
Signature · 1 line
2.38.1.431.g37b22c650d-goog
PATCH 3/3 x86/kasan: Populate shadow for shared chunk of the CPU entry area
In reply to Sean ChristophersonPopuplate the shadow for the shared portion of the CPU entry area, i.e. the read-only IDT mapping, during KASAN initialization. A recent change modified KASAN to map the per-CPU areas on-demand, but forgot to keep a shadow for the common area that is shared amongst all CPUs.
Map the common area in KASAN init instead of letting idt_map_in_cea() do the dirty work so that it Just Works in the unlikely event more shared data is shoved into the CPU entry area.
The bug manifests as a not-present #PF when software attempts to lookup an IDT entry, e.g. when KVM is handling IRQs on Intel CPUs (KVM performs direct CALL to the IRQ handler to avoid the overhead of INTn):
BUG: unable to handle page fault for address: fffffbc0000001d8 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 16c03a067 P4D 16c03a067 PUD 0 Oops: 0000 [#1] PREEMPT SMP KASAN CPU: 5 PID: 901 Comm: repro Tainted: G W 6.1.0-rc3+ #410 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:kasan_check_range+0xdf/0x190 vmx_handle_exit_irqoff+0x152/0x290 [kvm_intel] vcpu_run+0x1d89/0x2bd0 [kvm] kvm_arch_vcpu_ioctl_run+0x3ce/0xa70 [kvm] kvm_vcpu_ioctl+0x349/0x900 [kvm] __x64_sys_ioctl+0xb8/0xf0 do_syscall_64+0x2b/0x50 entry_SYSCALL_64_after_hwframe+0x46/0xb0
--- arch/x86/mm/kasan_init_64.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
arch/x86/mm/kasan_init_64.c +11 -1
@@ -341,7 +341,7 @@ void __init kasan_populate_shadow_for_vaddr(void *va, size_t size, int nid)void __init kasan_init (void ) { -unsigned long shadow_cea_begin, shadow_cea_end; +unsigned long shadow_cea_begin, shadow_cea_per_cpu_begin, shadow_cea_end;int i;memcpy (early_top_pgt, init_top_pgt,sizeof (early_top_pgt));
@@ -384,6 +384,7 @@ void __init kasan_init(void) } shadow_cea_begin =kasan_mem_to_shadow_align_down (CPU_ENTRY_AREA_BASE ); + shadow_cea_per_cpu_begin =kasan_mem_to_shadow_align_up (CPU_ENTRY_AREA_PER_CPU ); shadow_cea_end =kasan_mem_to_shadow_align_up (CPU_ENTRY_AREA_BASE +CPU_ENTRY_AREA_MAP_SIZE );
@@ -409,6 +410,15 @@ void __init kasan_init(void)kasan_mem_to_shadow ((void *)VMALLOC_END +1 ), (void *)shadow_cea_begin); +/* +* Populate the shadow for the shared portion of the CPU entry area. +* Shadows for the per-CPU areas are mapped on-demand, as each CPU's +* area is randomly placed somewhere in the 512GiB range and mapping +* the entire 512GiB range is prohibitively expensive. +*/ +kasan_populate_shadow (shadow_cea_begin, + shadow_cea_per_cpu_begin,0 ); +kasan_populate_early_shadow ((void *)shadow_cea_end,kasan_mem_to_shadow ((void *)__START_KERNEL_map));
Signature · 1 line
2.38.1.431.g37b22c650d-goog
Andrey Ryabinin
maintainerRe: [PATCH 3/3] x86/kasan: Populate shadow for shared chunk of the CPU entry area
Quoted 61 lines · Sean Christopherson
Popuplate the shadow for the shared portion of the CPU entry area, i.e. the read-only IDT mapping, during KASAN initialization. A recent change modified KASAN to map the per-CPU areas on-demand, but forgot to keep a shadow for the common area that is shared amongst all CPUs.
Map the common area in KASAN init instead of letting
idt_map_in_cea()do the dirty work so that it Just Works in the unlikely event more shared data is shoved into the CPU entry area.The bug manifests as a not-present #PF when software attempts to lookup an IDT entry, e.g. when KVM is handling IRQs on Intel CPUs (KVM performs direct CALL to the IRQ handler to avoid the overhead of INTn):
BUG: unable to handle page fault for address: fffffbc0000001d8 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 16c03a067 P4D 16c03a067 PUD 0 Oops: 0000 [#1] PREEMPT SMP KASAN CPU: 5 PID: 901 Comm: repro Tainted: G W 6.1.0-rc3+ #410 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:kasan_check_range+0xdf/0x190 vmx_handle_exit_irqoff+0x152/0x290 [kvm_intel] vcpu_run+0x1d89/0x2bd0 [kvm] kvm_arch_vcpu_ioctl_run+0x3ce/0xa70 [kvm] kvm_vcpu_ioctl+0x349/0x900 [kvm] __x64_sys_ioctl+0xb8/0xf0 do_syscall_64+0x2b/0x50 entry_SYSCALL_64_after_hwframe+0x46/0xb0Fixes 9fd429c28073 ("x86/kasan: Map shadow for percpu pages on demand")Reported-by [email protected]Cc Andrey Ryabinin <[email protected]>Signed-off-by Sean Christopherson <[email protected]>--- arch/x86/mm/kasan_init_64.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)arch/x86/mm/kasan_init_64.c +2
-1@@ -341,7 +341,7 @@ void __init kasan_populate_shadow_for_vaddr(void *va, size_t size, int nid)void __init kasan_init (void ) { -unsigned long shadow_cea_begin, shadow_cea_end; +unsigned long shadow_cea_begin, shadow_cea_per_cpu_begin, shadow_cea_end;int i;memcpy (early_top_pgt, init_top_pgt,sizeof (early_top_pgt));@@ -384,6 +384,7 @@ void __init kasan_init(void) } shadow_cea_begin =kasan_mem_to_shadow_align_down (CPU_ENTRY_AREA_BASE ); + shadow_cea_per_cpu_begin =kasan_mem_to_shadow_align_up (CPU_ENTRY_AREA_PER_CPU ); shadow_cea_end =kasan_mem_to_shadow_align_up (CPU_ENTRY_AREA_BASE +CPU_ENTRY_AREA_MAP_SIZE );@@ -409,6 +410,15 @@ void __init kasan_init(void)kasan_mem_to_shadow ((void *)VMALLOC_END +1 ), (void *)shadow_cea_begin);
arch/x86/mm/kasan_init_64.c void __init kasan_init(void) · line 421
Quoted 10 lines
arch/x86/mm/kasan_init_64.c +9
-0+/* +* Populate the shadow for the shared portion of the CPU entry area. +* Shadows for the per-CPU areas are mapped on-demand, as each CPU's +* area is randomly placed somewhere in the 512GiB range and mapping +* the entire 512GiB range is prohibitively expensive. +*/ +kasan_populate_shadow (shadow_cea_begin, + shadow_cea_per_cpu_begin,0 ); +
I think we can extend the kasan_populate_early_shadow() call above up to shadow_cea_per_cpu_begin point, instead of this.populate_early_shadow() maps single RO zeroed page. No one should write to the shadow for IDT. KASAN only needs writable shadow for linear mapping/stacks/vmalloc/global variables.
Quoted 3 lines
arch/x86/mm/kasan_init_64.c +0
-0kasan_populate_early_shadow ((void *)shadow_cea_end,kasan_mem_to_shadow ((void *)__START_KERNEL_map));
Re: [PATCH 3/3] x86/kasan: Populate shadow for shared chunk of the CPU entry area
Quoted 6 lines · Andrey Ryabinin
Quoted 4 lines · Sean Christopherson
arch/x86/mm/kasan_init_64.c +0
-0@@ -409,6 +410,15 @@ void __init kasan_init(void)kasan_mem_to_shadow ((void *)VMALLOC_END +1 ), (void *)shadow_cea_begin);
Quoted 14 lines
arch/x86/mm/kasan_init_64.c void __init kasan_init(void) · line 421
Quoted 9 lines
arch/x86/mm/kasan_init_64.c +9
-0+/* +* Populate the shadow for the shared portion of the CPU entry area. +* Shadows for the per-CPU areas are mapped on-demand, as each CPU's +* area is randomly placed somewhere in the 512GiB range and mapping +* the entire 512GiB range is prohibitively expensive. +*/ +kasan_populate_shadow (shadow_cea_begin, + shadow_cea_per_cpu_begin,0 ); +I think we can extend the
kasan_populate_early_shadow()call above up toshadow_cea_per_cpu_beginpoint, instead of this.populate_early_shadow()maps single RO zeroed page. No one should write to the shadow for IDT. KASAN only needs writable shadow for linear mapping/stacks/vmalloc/global variables.
Is that the only difference between the "early" and "normal" variants? If so, renaming them to kasan_populate_ro_shadow() vs. kasan_populate_rw_shadow() would make this code much more intuitive for non-KASAN folks.
Quoted 4 lines
Quoted 4 lines
arch/x86/mm/kasan_init_64.c +0
-0kasan_populate_early_shadow ((void *)shadow_cea_end,kasan_mem_to_shadow ((void *)__START_KERNEL_map));
Andrey Ryabinin
maintainerRe: [PATCH 3/3] x86/kasan: Populate shadow for shared chunk of the CPU entry area
Quoted 22 lines · Sean Christopherson
Quoted 6 lines · Andrey Ryabinin
Quoted 4 lines · Sean Christopherson
arch/x86/mm/kasan_init_64.c +0
-0@@ -409,6 +410,15 @@ void __init kasan_init(void)kasan_mem_to_shadow ((void *)VMALLOC_END +1 ), (void *)shadow_cea_begin);Quoted 14 lines
arch/x86/mm/kasan_init_64.c void __init kasan_init(void) · line 421
Quoted 9 lines
arch/x86/mm/kasan_init_64.c +9
-0+/* +* Populate the shadow for the shared portion of the CPU entry area. +* Shadows for the per-CPU areas are mapped on-demand, as each CPU's +* area is randomly placed somewhere in the 512GiB range and mapping +* the entire 512GiB range is prohibitively expensive. +*/ +kasan_populate_shadow (shadow_cea_begin, + shadow_cea_per_cpu_begin,0 ); +I think we can extend the
kasan_populate_early_shadow()call above up toshadow_cea_per_cpu_beginpoint, instead of this.populate_early_shadow()maps single RO zeroed page. No one should write to the shadow for IDT. KASAN only needs writable shadow for linear mapping/stacks/vmalloc/global variables.
Quoted 1 line
Is that the only difference between the "early" and "normal" variants?
It is. kasan_populate_shadow() allocates new memory and maps it, while the "early" one maps 'kasan_early_shadow_page'
Quoted 3 lines
If so, renaming them to
kasan_populate_ro_shadow()vs.kasan_populate_rw_shadow()would
make this code much more intuitive for non-KASAN folks.
Agreed.
Re: [PATCH 3/3] x86/kasan: Populate shadow for shared chunk of the CPU entry area
In reply to Andrey RyabininQuoted 6 lines · Andrey Ryabinin
Quoted 4 lines · Sean Christopherson
arch/x86/mm/kasan_init_64.c +0
-0@@ -409,6 +410,15 @@ void __init kasan_init(void)kasan_mem_to_shadow ((void *)VMALLOC_END +1 ), (void *)shadow_cea_begin);
Quoted 14 lines
arch/x86/mm/kasan_init_64.c void __init kasan_init(void) · line 421
Quoted 9 lines
arch/x86/mm/kasan_init_64.c +9
-0+/* +* Populate the shadow for the shared portion of the CPU entry area. +* Shadows for the per-CPU areas are mapped on-demand, as each CPU's +* area is randomly placed somewhere in the 512GiB range and mapping +* the entire 512GiB range is prohibitively expensive. +*/ +kasan_populate_shadow (shadow_cea_begin, + shadow_cea_per_cpu_begin,0 ); +I think we can extend the
kasan_populate_early_shadow()call above up toshadow_cea_per_cpu_beginpoint, instead of this.populate_early_shadow()maps single RO zeroed page. No one should write to the shadow for IDT. KASAN only needs writable shadow for linear mapping/stacks/vmalloc/global variables.
Any objection to simply converting this to use kasan_populate_early_shadow(), i.e. to keeping a separate "populate" call for the CPU entry area? Purely so that it's more obvious that a small portion of the overall CPU entry area is mapped during init.