x86/boot/64: Fix crash if kernel images crosses page table boundary
PATCH 1/1 x86/boot/64: Fix crash if kernel images crosses page table boundary
Kernel that boots in 5-level paging mode crashes in small percentage of cases if KASLR is enabled.
This issue was tracked down to the case when the kernel image unpack in the way it crosses 1G boundary. The crash was due to overrun of PMD page table in __startup_64() and corruption of P4D page table allocated next to it.
The issue is not visible with 4-level paging as we don't use the P4D page table.
The root cause is that the handling of page table wraparound was lost when part of startup_64() was rewritten in C.
Restore the handling. It also fixes the issue when the kernel image put across 512G and, for 5-level paging, 64T boundary.
--- arch/x86/kernel/head64.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
arch/x86/kernel/head64.c +8 -8
@@ -190,18 +190,18 @@ unsigned long __head __startup_64(unsigned long physaddr, pgd[i +0 ] = (pgdval_t )p4d + pgtable_flags; pgd[i +1 ] = (pgdval_t )p4d + pgtable_flags; - i = (physaddr >>P4D_SHIFT ) %PTRS_PER_P4D ; - p4d[i +0 ] = (pgdval_t )pud + pgtable_flags; - p4d[i +1 ] = (pgdval_t )pud + pgtable_flags; + i = physaddr >>P4D_SHIFT ; + p4d[(i +0 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; + p4d[(i +1 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; }else { i = (physaddr >>PGDIR_SHIFT ) %PTRS_PER_PGD ; pgd[i +0 ] = (pgdval_t )pud + pgtable_flags; pgd[i +1 ] = (pgdval_t )pud + pgtable_flags; } - i = (physaddr >>PUD_SHIFT ) %PTRS_PER_PUD ; - pud[i +0 ] = (pudval_t )pmd + pgtable_flags; - pud[i +1 ] = (pudval_t )pmd + pgtable_flags; + i = physaddr >>PUD_SHIFT ; + pud[(i +0 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; + pud[(i +1 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;/* Filter out unsupported __PAGE_KERNEL_* bits: */
@@ -211,8 +211,8 @@ unsigned long __head __startup_64(unsigned long physaddr, pmd_entry += physaddr;for (i =0 ; i <DIV_ROUND_UP (_end - _text,PMD_SIZE ); i++) { -int idx = i + (physaddr >>PMD_SHIFT ) %PTRS_PER_PMD ; - pmd[idx] = pmd_entry + i *PMD_SIZE ; +int idx = i + (physaddr >>PMD_SHIFT );; + pmd[idx %PTRS_PER_PMD ] = pmd_entry + i *PMD_SIZE ; }/*
Signature · 1 line
2.21.0
Quoted 26 lines · Kirill A. Shutemov
arch/x86/kernel/head64.c +6
-6@@ -190,18 +190,18 @@ unsigned long __head __startup_64(unsigned long physaddr, pgd[i +0 ] = (pgdval_t )p4d + pgtable_flags; pgd[i +1 ] = (pgdval_t )p4d + pgtable_flags; - i = (physaddr >>P4D_SHIFT ) %PTRS_PER_P4D ; - p4d[i +0 ] = (pgdval_t )pud + pgtable_flags; - p4d[i +1 ] = (pgdval_t )pud + pgtable_flags; + i = physaddr >>P4D_SHIFT ; + p4d[(i +0 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; + p4d[(i +1 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; }else { i = (physaddr >>PGDIR_SHIFT ) %PTRS_PER_PGD ; pgd[i +0 ] = (pgdval_t )pud + pgtable_flags; pgd[i +1 ] = (pgdval_t )pud + pgtable_flags; } - i = (physaddr >>PUD_SHIFT ) %PTRS_PER_PUD ; - pud[i +0 ] = (pudval_t )pmd + pgtable_flags; - pud[i +1 ] = (pudval_t )pmd + pgtable_flags; + i = physaddr >>PUD_SHIFT ; + pud[(i +0 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; + pud[(i +1 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;/* Filter out unsupported __PAGE_KERNEL_* bits: */
arch/x86/kernel/head64.c unsigned long __head __startup_64(unsigned long physaddr, · line 214
Quoted 6 lines
arch/x86/kernel/head64.c +1
-2@@ -211,8 +211,8 @@ unsigned long __head __startup_64(unsigned long physaddr, pmd_entry += physaddr;for (i =0 ; i <DIV_ROUND_UP (_end - _text,PMD_SIZE ); i++) { -int idx = i + (physaddr >>PMD_SHIFT ) %PTRS_PER_PMD ; - pmd[idx] = pmd_entry + i *PMD_SIZE ; +int idx = i + (physaddr >>PMD_SHIFT );;
double semicolon
arch/x86/kernel/head64.c unsigned long __head __startup_64(unsigned long physaddr, · line 215
Quoted 1 line
arch/x86/kernel/head64.c +1
-0+ pmd[idx %PTRS_PER_PMD ] = pmd_entry + i *PMD_SIZE ;
This part is functionally equivivalent. So what's the value of this change?
Thanks,
tglx
Quoted 29 lines · Thomas Gleixner
Quoted 28 lines · Kirill A. Shutemov
arch/x86/kernel/head64.c +6
-6@@ -190,18 +190,18 @@ unsigned long __head __startup_64(unsigned long physaddr, pgd[i +0 ] = (pgdval_t )p4d + pgtable_flags; pgd[i +1 ] = (pgdval_t )p4d + pgtable_flags; - i = (physaddr >>P4D_SHIFT ) %PTRS_PER_P4D ; - p4d[i +0 ] = (pgdval_t )pud + pgtable_flags; - p4d[i +1 ] = (pgdval_t )pud + pgtable_flags; + i = physaddr >>P4D_SHIFT ; + p4d[(i +0 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; + p4d[(i +1 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; }else { i = (physaddr >>PGDIR_SHIFT ) %PTRS_PER_PGD ; pgd[i +0 ] = (pgdval_t )pud + pgtable_flags; pgd[i +1 ] = (pgdval_t )pud + pgtable_flags; } - i = (physaddr >>PUD_SHIFT ) %PTRS_PER_PUD ; - pud[i +0 ] = (pudval_t )pmd + pgtable_flags; - pud[i +1 ] = (pudval_t )pmd + pgtable_flags; + i = physaddr >>PUD_SHIFT ; + pud[(i +0 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; + pud[(i +1 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;/* Filter out unsupported __PAGE_KERNEL_* bits: */ @@ -211,8 +211,8 @@ unsigned long __head __startup_64(unsigned long physaddr, pmd_entry += physaddr;
Quoted 6 lines
arch/x86/kernel/head64.c unsigned long __head __startup_64(unsigned long physaddr, · line 214
Quoted 4 lines
arch/x86/kernel/head64.c +1
-2for (i =0 ; i <DIV_ROUND_UP (_end - _text,PMD_SIZE ); i++) { -int idx = i + (physaddr >>PMD_SHIFT ) %PTRS_PER_PMD ; - pmd[idx] = pmd_entry + i *PMD_SIZE ; +int idx = i + (physaddr >>PMD_SHIFT );;double semicolon
Will fix.
Quoted 3 lines
arch/x86/kernel/head64.c unsigned long __head __startup_64(unsigned long physaddr, · line 215
Quoted 1 line
arch/x86/kernel/head64.c +1
-0+ pmd[idx %PTRS_PER_PMD ] = pmd_entry + i *PMD_SIZE ;This part is functionally equivivalent. So what's the value of this change?
Precedence of operators were broken
idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;
reads by compiler as
idx = i + ((physaddr >> PMD_SHIFT) % PTRS_PER_PMD);
not as
idx = (i + (physaddr >> PMD_SHIFT)) % PTRS_PER_PMD;
Therefore 'idx' can become >= PTRS_PER_PMD.
Signature · 1 line
Kirill A. Shutemov
Quoted 12 lines · Kirill A. Shutemov
Quoted 3 lines · Thomas Gleixner
arch/x86/kernel/head64.c unsigned long __head __startup_64(unsigned long physaddr, · line 215
Quoted 1 line
arch/x86/kernel/head64.c +1
-0+ pmd[idx %PTRS_PER_PMD ] = pmd_entry + i *PMD_SIZE ;This part is functionally equivivalent. So what's the value of this change?
Precedence of operators were broken
idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;reads by compiler as
Quoted 7 lines
idx = i + ((physaddr >> PMD_SHIFT) % PTRS_PER_PMD);not as
idx = (i + (physaddr >> PMD_SHIFT)) % PTRS_PER_PMD;Therefore 'idx' can become >=
PTRS_PER_PMD.
Indeed. Please mention it in the change log. I did not spot it right away.
Thanks,
tglx
Quoted 15 lines · Thomas Gleixner
Quoted 14 lines · Kirill A. Shutemov
Quoted 3 lines · Thomas Gleixner
arch/x86/kernel/head64.c unsigned long __head __startup_64(unsigned long physaddr, · line 215
Quoted 1 line
arch/x86/kernel/head64.c +1
-0+ pmd[idx %PTRS_PER_PMD ] = pmd_entry + i *PMD_SIZE ;This part is functionally equivivalent. So what's the value of this change?
Precedence of operators were broken
idx = i + (physaddr >> PMD_SHIFT) % PTRS_PER_PMD;reads by compiler as
idx = i + ((physaddr >> PMD_SHIFT) % PTRS_PER_PMD);
Quoted 7 lines
Quoted 5 lines
not as
idx = (i + (physaddr >> PMD_SHIFT)) % PTRS_PER_PMD;Therefore 'idx' can become >=
PTRS_PER_PMD.Indeed. Please mention it in the change log. I did not spot it right away.
I fixed it up already. No need to resend.
Thanks,
tglx
tip-bot for Kirill A. Shutemov
merge bot[tip:x86/urgent] x86/boot/64: Fix crash if kernel image crosses page table boundary
In reply to Kirill A. Shutemovx86/boot/64: Fix crash if kernel image crosses page table boundary
Commit-ID: 81c7ed296dcd02bc0b4488246d040e03e633737a Gitweb: https://git.kernel.org/tip/81c7ed296dcd02bc0b4488246d040e03e633737a Author: Kirill A. Shutemov <[email protected]> AuthorDate: Thu, 20 Jun 2019 14:23:45 +0300 Committer: Thomas Gleixner <[email protected]> CommitDate: Wed, 26 Jun 2019 07:25:09 +0200
x86/boot/64: Fix crash if kernel image crosses page table boundary
A kernel which boots in 5-level paging mode crashes in a small percentage of cases if KASLR is enabled.
This issue was tracked down to the case when the kernel image unpacks in a way that it crosses an 1G boundary. The crash is caused by an overrun of the PMD page table in __startup_64() and corruption of P4D page table allocated next to it. This particular issue is not visible with 4-level paging as P4D page tables are not used.
But the P4D and the PUD calculation have similar problems.
The PMD index calculation is wrong due to operator precedence, which fails to confine the PMDs in the PMD array on wrap around.
The P4D calculation for 5-level paging and the PUD calculation calculate the first index correctly, but then blindly increment it which causes the same issue when a kernel image is located across a 512G and for 5-level paging across a 46T boundary.
This wrap around mishandling was introduced when these parts moved from assembly to C.
Restore it to the correct behaviour.
--- arch/x86/kernel/head64.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-)
arch/x86/kernel/head64.c +9 -8
@@ -190,18 +190,18 @@ unsigned long __head __startup_64(unsigned long physaddr, pgd[i +0 ] = (pgdval_t )p4d + pgtable_flags; pgd[i +1 ] = (pgdval_t )p4d + pgtable_flags; - i = (physaddr >>P4D_SHIFT ) %PTRS_PER_P4D ; - p4d[i +0 ] = (pgdval_t )pud + pgtable_flags; - p4d[i +1 ] = (pgdval_t )pud + pgtable_flags; + i = physaddr >>P4D_SHIFT ; + p4d[(i +0 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; + p4d[(i +1 ) %PTRS_PER_P4D ] = (pgdval_t )pud + pgtable_flags; }else { i = (physaddr >>PGDIR_SHIFT ) %PTRS_PER_PGD ; pgd[i +0 ] = (pgdval_t )pud + pgtable_flags; pgd[i +1 ] = (pgdval_t )pud + pgtable_flags; } - i = (physaddr >>PUD_SHIFT ) %PTRS_PER_PUD ; - pud[i +0 ] = (pudval_t )pmd + pgtable_flags; - pud[i +1 ] = (pudval_t )pmd + pgtable_flags; + i = physaddr >>PUD_SHIFT ; + pud[(i +0 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; + pud[(i +1 ) %PTRS_PER_PUD ] = (pudval_t )pmd + pgtable_flags; pmd_entry = __PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL;/* Filter out unsupported __PAGE_KERNEL_* bits: */
@@ -211,8 +211,9 @@ unsigned long __head __startup_64(unsigned long physaddr, pmd_entry += physaddr;for (i =0 ; i <DIV_ROUND_UP (_end - _text,PMD_SIZE ); i++) { -int idx = i + (physaddr >>PMD_SHIFT ) %PTRS_PER_PMD ; - pmd[idx] = pmd_entry + i *PMD_SIZE ; +int idx = i + (physaddr >>PMD_SHIFT ); + + pmd[idx %PTRS_PER_PMD ] = pmd_entry + i *PMD_SIZE ; }/*