x86/paravirt: Rework paravirt patching
Subsystems (2)
cover x86/paravirt: Rework paravirt patching
Andi tried to make the paravirt patch magic work nicely with LTO. It turned out that the DEF_NATIVE() macro is clever but fragile. Even without LTO it works by chance and not by design.
As the paravirt patch tables are not changed frequently it's reasonable to convert them to regular data which is initialized with byte sequences.
Looking at Andi's attempt to change the existing code made me realize that there is no real reason to have two variants for 32 and 64 bit which just differ slightly.
The following series unifies the 32/64bit implementation and converts it to use regular data at the end.
Thanks,
tglx
8<--------------
a/arch/x86/kernel/paravirt_patch_32.c | 67 ------------------ a/arch/x86/kernel/paravirt_patch_64.c | 75 -------------------- arch/x86/include/asm/paravirt_types.h | 4 - arch/x86/kernel/Makefile | 4 - b/arch/x86/kernel/paravirt_patch.c | 126 ++++++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 148 deletions(-)
PATCH 1/3 x86/paravirt: Remove bogus extern declarations
These functions are already declared in asm/paravirt.h
--- arch/x86/kernel/paravirt_patch_32.c | 3 --- arch/x86/kernel/paravirt_patch_64.c | 3 --- 2 files changed, 6 deletions(-)
arch/x86/kernel/paravirt_patch_32.c +0 -3
@@ -23,9 +23,6 @@ DEF_NATIVE(lock, queued_spin_unlock, "moDEF_NATIVE (lock, vcpu_is_preempted,"xor %eax, %eax" );#endif -extern bool pv_is_native_spin_unlock (void ); -extern bool pv_is_native_vcpu_is_preempted (void ); -unsigned native_patch (u8 type,void *ibuf,unsigned long addr,unsigned len) {#define PATCH_SITE (ops, x) \
arch/x86/kernel/paravirt_patch_64.c +0 -3
@@ -29,9 +29,6 @@ DEF_NATIVE(lock, queued_spin_unlock, "moDEF_NATIVE (lock, vcpu_is_preempted,"xor %eax, %eax" );#endif -extern bool pv_is_native_spin_unlock (void ); -extern bool pv_is_native_vcpu_is_preempted (void ); -unsigned native_patch (u8 type,void *ibuf,unsigned long addr,unsigned len) {#define PATCH_SITE (ops, x) \
tip-bot for Thomas Gleixner
merge bot[tip:x86/paravirt] x86/paravirt: Remove bogus extern declarations
x86/paravirt: Remove bogus extern declarations
Commit-ID: 465f81857c16ae17f461e4738ceb1f4f8cce2077 Gitweb: https://git.kernel.org/tip/465f81857c16ae17f461e4738ceb1f4f8cce2077 Author: Thomas Gleixner <[email protected]> AuthorDate: Wed, 24 Apr 2019 15:41:16 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 08:45:01 +0200
x86/paravirt: Remove bogus extern declarations
These functions are already declared in asm/paravirt.h
--- arch/x86/kernel/paravirt_patch_32.c | 3 --- arch/x86/kernel/paravirt_patch_64.c | 3 --- 2 files changed, 6 deletions(-)
arch/x86/kernel/paravirt_patch_32.c +0 -3
@@ -23,9 +23,6 @@ DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%eax)");DEF_NATIVE (lock, vcpu_is_preempted,"xor %eax, %eax" );#endif -extern bool pv_is_native_spin_unlock (void ); -extern bool pv_is_native_vcpu_is_preempted (void ); -unsigned native_patch (u8 type,void *ibuf,unsigned long addr,unsigned len) {#define PATCH_SITE (ops, x) \
arch/x86/kernel/paravirt_patch_64.c +0 -3
@@ -29,9 +29,6 @@ DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%rdi)");DEF_NATIVE (lock, vcpu_is_preempted,"xor %eax, %eax" );#endif -extern bool pv_is_native_spin_unlock (void ); -extern bool pv_is_native_vcpu_is_preempted (void ); -unsigned native_patch (u8 type,void *ibuf,unsigned long addr,unsigned len) {#define PATCH_SITE (ops, x) \
tip-bot for Thomas Gleixner
merge bot[tip:x86/paravirt] x86/paravirt: Remove bogus extern declarations
In reply to Thomas Gleixnerx86/paravirt: Remove bogus extern declarations
Commit-ID: e05196401657cff3178dc392b739e520b26d4aef Gitweb: https://git.kernel.org/tip/e05196401657cff3178dc392b739e520b26d4aef Author: Thomas Gleixner <[email protected]> AuthorDate: Wed, 24 Apr 2019 15:41:16 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 11:35:55 +0200
x86/paravirt: Remove bogus extern declarations
These functions are already declared in asm/paravirt.h
--- arch/x86/kernel/paravirt_patch_32.c | 3 --- arch/x86/kernel/paravirt_patch_64.c | 3 --- 2 files changed, 6 deletions(-)
arch/x86/kernel/paravirt_patch_32.c +0 -3
@@ -23,9 +23,6 @@ DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%eax)");DEF_NATIVE (lock, vcpu_is_preempted,"xor %eax, %eax" );#endif -extern bool pv_is_native_spin_unlock (void ); -extern bool pv_is_native_vcpu_is_preempted (void ); -unsigned native_patch (u8 type,void *ibuf,unsigned long addr,unsigned len) {#define PATCH_SITE (ops, x) \
arch/x86/kernel/paravirt_patch_64.c +0 -3
@@ -29,9 +29,6 @@ DEF_NATIVE(lock, queued_spin_unlock, "movb $0, (%rdi)");DEF_NATIVE (lock, vcpu_is_preempted,"xor %eax, %eax" );#endif -extern bool pv_is_native_spin_unlock (void ); -extern bool pv_is_native_vcpu_is_preempted (void ); -unsigned native_patch (u8 type,void *ibuf,unsigned long addr,unsigned len) {#define PATCH_SITE (ops, x) \
PATCH 2/3 x86/paravirt: Unify 32/64 bit patch code
In reply to Thomas GleixnerLarge parts of these two files are identical. Merge them together.
--- arch/x86/kernel/Makefile | 4 - arch/x86/kernel/paravirt_patch.c | 106 ++++++++++++++++++++++++++++++++++++ arch/x86/kernel/paravirt_patch_32.c | 64 --------------------- arch/x86/kernel/paravirt_patch_64.c | 72 ------------------------ 4 files changed, 108 insertions(+), 138 deletions(-)
arch/x86/kernel/Makefile +2 -2
@@ -30,7 +30,7 @@ KASAN_SANITIZE_paravirt.o := n OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y OBJECT_FILES_NON_STANDARD_test_nx.o := y -OBJECT_FILES_NON_STANDARD_paravirt_patch_$(BITS).o := y +OBJECT_FILES_NON_STANDARD_paravirt_patch.o := y ifdef CONFIG_FRAME_POINTER OBJECT_FILES_NON_STANDARD_ftrace_$(BITS).o := y
@@ -112,7 +112,7 @@ obj-$(CONFIG_AMD_NB) += amd_nb.o obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o -obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o +obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch.o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o
arch/x86/kernel/paravirt_patch.c new file +106 -0
106 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_32.c deleted +0 -64
64 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_64.c deleted +0 -72
72 lines not on this page · raw message
tip-bot for Thomas Gleixner
merge bot[tip:x86/paravirt] x86/paravirt: Unify 32/64 bit patch code
x86/paravirt: Unify 32/64 bit patch code
Commit-ID: 92c814ed5f39d0f03d8cc432449a07f82e2c8a73 Gitweb: https://git.kernel.org/tip/92c814ed5f39d0f03d8cc432449a07f82e2c8a73 Author: Thomas Gleixner <[email protected]> AuthorDate: Wed, 24 Apr 2019 15:41:17 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 08:45:02 +0200
x86/paravirt: Unify 32/64 bit patch code
Large parts of these two files are identical. Merge them together.
---
arch/x86/kernel/Makefile | 4 +-
.../{paravirt_patch_64.c => paravirt_patch.c} | 62 ++++++++++++++++-----
arch/x86/kernel/paravirt_patch_32.c | 64 ----------------------
3 files changed, 50 insertions(+), 80 deletions(-)arch/x86/kernel/Makefile +2 -2
@@ -30,7 +30,7 @@ KASAN_SANITIZE_paravirt.o := n OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y OBJECT_FILES_NON_STANDARD_test_nx.o := y -OBJECT_FILES_NON_STANDARD_paravirt_patch_$(BITS).o := y +OBJECT_FILES_NON_STANDARD_paravirt_patch.o := y ifdef CONFIG_FRAME_POINTER OBJECT_FILES_NON_STANDARD_ftrace_$(BITS).o := y
@@ -112,7 +112,7 @@ obj-$(CONFIG_AMD_NB) += amd_nb.o obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o -obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o +obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch.o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o
arch/x86/kernel/paravirt_patch.c renamed from arch/x86/kernel/paravirt_patch_64.c +48 -14
95 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_32.c deleted +0 -64
64 lines not on this page · raw message
tip-bot for Thomas Gleixner
merge bot[tip:x86/paravirt] x86/paravirt: Unify the 32/64 bit paravirt patching code
In reply to Thomas Gleixnerx86/paravirt: Unify the 32/64 bit paravirt patching code
Commit-ID: fb2af0712fe8831dc152b0b5dd8bc516970da336 Gitweb: https://git.kernel.org/tip/fb2af0712fe8831dc152b0b5dd8bc516970da336 Author: Thomas Gleixner <[email protected]> AuthorDate: Wed, 24 Apr 2019 15:41:17 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 12:00:44 +0200
x86/paravirt: Unify the 32/64 bit paravirt patching code
Large parts of these two files are identical. Merge them together.
---
arch/x86/kernel/Makefile | 4 +-
.../{paravirt_patch_64.c => paravirt_patch.c} | 62 ++++++++++++++++-----
arch/x86/kernel/paravirt_patch_32.c | 64 ----------------------
3 files changed, 50 insertions(+), 80 deletions(-)arch/x86/kernel/Makefile +2 -2
@@ -30,7 +30,7 @@ KASAN_SANITIZE_paravirt.o := n OBJECT_FILES_NON_STANDARD_relocate_kernel_$(BITS).o := y OBJECT_FILES_NON_STANDARD_test_nx.o := y -OBJECT_FILES_NON_STANDARD_paravirt_patch_$(BITS).o := y +OBJECT_FILES_NON_STANDARD_paravirt_patch.o := y ifdef CONFIG_FRAME_POINTER OBJECT_FILES_NON_STANDARD_ftrace_$(BITS).o := y
@@ -112,7 +112,7 @@ obj-$(CONFIG_AMD_NB) += amd_nb.o obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o -obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o +obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch.o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o
arch/x86/kernel/paravirt_patch.c renamed from arch/x86/kernel/paravirt_patch_64.c +48 -14
95 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_32.c deleted +0 -64
64 lines not on this page · raw message
PATCH 3/3 x86/paravirt: Replace paravirt patch asm magic
In reply to Thomas GleixnerThe magic macro DEF_NATIVE() in the paravirt patching code uses inline assembly to generate a data table for patching in the native instructions.
While clever this is falling apart with LTO and even aside of LTO the construct is just working by chance according to GCC folks.
Aside of that the tables are constant data and not some form of magic text.
As these constructs are not subject to frequent changes it is not a maintenance issue to convert them to regular data tables which are initialized with hex bytes.
Create a new set of macros and data structures to store the instruction sequences and convert the code over.
--- arch/x86/include/asm/paravirt_types.h | 4 arch/x86/kernel/paravirt_patch.c | 144 +++++++++++++++++++--------------- 2 files changed, 82 insertions(+), 66 deletions(-)
arch/x86/include/asm/paravirt_types.h +0 -4
@@ -370,10 +370,6 @@ extern struct paravirt_patch_template pv/* Simple instruction patching code. */ #define NATIVE_LABEL (a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t" -#define DEF_NATIVE (ops, name, code) \ - __visible extern const char start_##ops##_##name[], end_##ops##_##name[]; \ - asm(NATIVE_LABEL("start_", ops, name) code NATIVE_LABEL("end_", ops, name)) -unsigned paravirt_patch_ident_64 (void *insnbuf,unsigned len);unsigned paravirt_patch_default (u8 type,void *insnbuf,unsigned long addr,unsigned len);
arch/x86/kernel/paravirt_patch.c +82 -62
185 lines not on this page · raw message
Re: [patch 3/3] x86/paravirt: Replace paravirt patch asm magic
Quoted 38 lines · Thomas Gleixner
The magic macro
DEF_NATIVE()in the paravirt patching code uses inline assembly to generate a data table for patching in the native instructions.While clever this is falling apart with LTO and even aside of LTO the construct is just working by chance according to GCC folks.
Aside of that the tables are constant data and not some form of magic text.
As these constructs are not subject to frequent changes it is not a maintenance issue to convert them to regular data tables which are initialized with hex bytes.
Create a new set of macros and data structures to store the instruction sequences and convert the code over.
Reported-by Andi Kleen <[email protected]>Signed-off-by Thomas Gleixner <[email protected]>(quoted diff) +9
-9-# ifdef CONFIG_PARAVIRT_XXL -DEF_NATIVE(irq, irq_disable, "cli"); -DEF_NATIVE(irq, irq_enable, "sti"); -DEF_NATIVE(irq, restore_fl, "push %eax; popf"); -DEF_NATIVE(irq, save_fl, "pushf; pop %eax"); -DEF_NATIVE(cpu, iret, "iret"); -DEF_NATIVE(mmu, read_cr2, "mov %cr2, %eax"); -DEF_NATIVE(mmu, write_cr3, "mov %eax, %cr3"); -DEF_NATIVE(mmu, read_cr3, "mov %cr3, %eax"); +static const struct patch_xxl patch_data_xxl = { + .irq_irq_disable = { 0xfa }, // cli + .irq_irq_enable = { 0xfb }, // sti + .irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax + .mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax + .mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax +# ifdef CONFIG_X86_64 + .irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq + .mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3
Review
Quoted 10 lines
(quoted diff) +10
-0+ .cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd + .cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8, + 0x48, 0x0f, 0x07 }, // swapgs; sysretq + .cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs + .mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax +# else + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf + .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .cpu_iret = { 0xcf }, // iret +# endif
I think these open-coded hexa versions are somewhat fragile as well, how about putting these into a .S file and controlling the sections in an LTO safe manner there?
That will also allow us to write proper asm, and global labels can be used to extract the patchlets and their length?
Thanks,
Ingo
Re: [patch 3/3] x86/paravirt: Replace paravirt patch asm magic
Quoted 10 lines · Ingo Molnar
arch/x86/kernel/paravirt_patch.c line 57
Quoted 5 lines
arch/x86/kernel/paravirt_patch.c +5
-0+# else + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf + .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .cpu_iret = { 0xcf }, // iret +# endifI think these open-coded hexa versions are somewhat fragile as well, how about putting these into a .S file and controlling the sections in an LTO safe manner there?
Quoted 2 lines
That will also allow us to write proper asm, and global labels can be used to extract the patchlets and their length?
We are not changing these any other day and I really don't see a reason to have these things global just because.
Thanks,
tglx
Juergen Gross
maintainerRe: [patch 3/3] x86/paravirt: Replace paravirt patch asm magic
Quoted 11 lines · Thomas Gleixner
Quoted 10 lines · Ingo Molnar
arch/x86/kernel/paravirt_patch.c line 57
Quoted 5 lines
arch/x86/kernel/paravirt_patch.c +5
-0+# else + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf + .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .cpu_iret = { 0xcf }, // iret +# endifI think these open-coded hexa versions are somewhat fragile as well, how about putting these into a .S file and controlling the sections in an LTO safe manner there?
Quoted 5 lines
Quoted 2 lines
That will also allow us to write proper asm, and global labels can be used to extract the patchlets and their length?
We are not changing these any other day and I really don't see a reason to have these things global just because.
What about generating the hex values from the Makefile using a temporary .S file? This would make the result easy verifiable and I don't think setting this up is too complicated.
Juergen
PATCH 1/1 x86/paravirt: Match paravirt patchlet field definition ordering to initialization ordering
In reply to Thomas GleixnerQuoted 11 lines · Thomas Gleixner
Quoted 10 lines · Ingo Molnar
Review
Quoted 5 lines
(quoted diff) +5
-0+# else + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf + .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .cpu_iret = { 0xcf }, // iret +# endifI think these open-coded hexa versions are somewhat fragile as well, how about putting these into a .S file and controlling the sections in an LTO safe manner there?
Quoted 5 lines
Quoted 2 lines
That will also allow us to write proper asm, and global labels can be used to extract the patchlets and their length?
We are not changing these any other day and I really don't see a reason to have these things global just because.
Firstly, I'm not sure I understand the objection to using symbols: in an average distro kernel we have more than 75,000 symbols - as long as the namespace is clear they are there to be used, there's nothing wrong with them per se.
Secondly, my main point is that putting these code patching sequences into the .S it's easy to verify that the instructions and patchlets are what we claim them to be.
Right now they are randomly ordered data tables that don't match to the source code.
I know, because I tried. :-)
Here's the objdump -D output of the PATCH_XXL data table:
0000000000000010 <patch_data_xxl>:
10: fa cli 11: fb sti 12: 57 push %rdi 13: 9d popfq 14: 9c pushfq 15: 58 pop %rax 16: 0f 20 d0 mov %cr2,%rax 19: 0f 20 d8 mov %cr3,%rax 1c: 0f 22 df mov %rdi,%cr3 1f: 0f 09 wbinvd 21: 0f 01 f8 swapgs 24: 48 0f 07 sysretq 27: 0f 01 f8 swapgs 2a: 48 89 f8 mov %rdi,%rax
Note how this doesn't match up to the source code:
static const struct patch_xxl patch_data_xxl = {
.irq_irq_disable = { 0xfa }, // cli
.irq_irq_enable = { 0xfb }, // sti
.irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax
.mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax
.mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax
# ifdef CONFIG_X86_64
.irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq
.mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3
.cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd
.cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8,
0x48, 0x0f, 0x07 }, // swapgs; sysretq
.cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs
.mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax
# else
.irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf
.mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3
.cpu_iret = { 0xcf }, // iret
# endif
};Note how they are reordered: in the generated code .irq_restore_fl comes before .irq_save_fl, etc. This is because the field ordering in struct patch_xxl does not match the initialization ordering of patch_data_xxl.
Third, beyond readability there's another advantage of my suggested approach as well: for example that way we could verify the passed in length with the patchlet length. Right now it's completely unverified:
case PARAVIRT_PATCH(ops.m): \
return PATCH(data, ops##_##m, ibuf, len)
right now we don't check whether the 'len' passed in by the usage site matches the actual structure field length.
Although maybe we could do that with your C space structure as well.
Anyway, no strong feelings and I didn't want to create extra work for you
- but I think at minimum we should do the patch below, which matches up the initialization order with the definition order - this at least makes the disassembly reviewable:
0000000000000010 <patch_data_xxl>:
10: fa cli 11: fb sti 12: 9c pushfq 13: 58 pop %rax 14: 0f 20 d0 mov %cr2,%rax 17: 0f 20 d8 mov %cr3,%rax 1a: 0f 22 df mov %rdi,%cr3 1d: 57 push %rdi 1e: 9d popfq 1f: 0f 09 wbinvd 21: 0f 01 f8 swapgs 24: 48 0f 07 sysretq 27: 0f 01 f8 swapgs 2a: 48 89 f8 mov %rdi,%rax
And yes, with that applied it verifies 100%. :-)
Thanks,
Ingo
arch/x86/kernel/paravirt_patch.c +3 -3
@@ -21,11 +21,11 @@struct patch_xxl {const unsigned char irq_irq_disable[1 ];const unsigned char irq_irq_enable[1 ]; -const unsigned char irq_restore_fl[2 ];const unsigned char irq_save_fl[2 ];const unsigned char mmu_read_cr2[3 ];const unsigned char mmu_read_cr3[3 ];const unsigned char mmu_write_cr3[3 ]; +const unsigned char irq_restore_fl[2 ];# ifdef CONFIG_X86_64 const unsigned char cpu_wbinvd[2 ];const unsigned char cpu_usergs_sysret64[6 ];
@@ -43,16 +43,16 @@ static const struct patch_xxl patch_data .mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax .mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax # ifdef CONFIG_X86_64 - .irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq .mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3 + .irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq .cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd .cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8, 0x48, 0x0f, 0x07 }, // swapgs; sysretq .cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs .mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax # else - .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf .cpu_iret = { 0xcf }, // iret # endif };
PATCH 1/1 x86/paravirt: Detect oversized patching bugs as they happen and BUG_ON() to avoid later crashes
Quoted 11 lines · Ingo Molnar
Third, beyond readability there's another advantage of my suggested approach as well: for example that way we could verify the passed in length with the patchlet length. Right now it's completely unverified:
case
PARAVIRT_PATCH(ops.m): \
returnPATCH(data, ops##_##m, ibuf, len)right now we don't check whether the 'len' passed in by the usage site matches the actual structure field length.
Although maybe we could do that with your C space structure as well.
So I was wrong here, got confused by the 'len' name which doesn't mean what it suggests: it's not the length of the patchlet, but the maximum/original length of the patch site - which we trim down in paravirt_patch_insns():
unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
const char *start, const char *end)
{
unsigned insn_len = end - start;
if (insn_len > len || start == NULL)
insn_len = len;
else
memcpy(insnbuf, start, insn_len);
return insn_len;
}What is the logic behind silently returning 'len' here and not copying anything?
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
Instead I think we should BUG_ON() that condition with the patch below - there's no way to continue successfully at that point.
I've tested this patch, with the vanilla kernel check never triggers, and if I intentionally increase the size of one of the patch templates to a too high value the assert triggers:
[ 0.164385] kernel BUG at arch/x86/kernel/paravirt.c:167!
Without this patch a broken kernel randomly crashes in later places, after the silent patching failure.
Thanks,
Ingo
arch/x86/kernel/paravirt.c +4 -4
@@ -163,10 +163,10 @@ unsigned paravirt_patch_insns(void *insn {unsigned insn_len = end - start; -if (insn_len > len || start ==NULL ) - insn_len = len; -else -memcpy (insnbuf, start, insn_len); +/* Alternative instruction is too large for the patch site and we cannot continue: */ +BUG_ON (insn_len > len || start ==NULL ); + +memcpy (insnbuf, start, insn_len);return insn_len; }
Re: [PATCH] x86/paravirt: Detect oversized patching bugs as they happen and BUG_ON() to avoid later crashes
Quoted 3 lines · Ingo Molnar
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
That's OK. The compiler emits an indirect CALL/JMP to the pv_ops structure contents. That _should_ stay valid and function correctly at all times.
Not patching should at the very least cause a WARN with RETPOLINE kernels though, we hard rely on the patching actually working and writing at least a direct call.
x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
That's OK. The compiler emits an indirect CALL/JMP to the
pv_opsstructure contents. That _should_ stay valid and function correctly at all times.
It might result in a correctly executing kernel in terms of code generation, but it doesn't result in a viable kernel: some of the places rely on the patching going through and don't know what to do when it doesn't and misbehave or crash in interesting ways.
Guess how I know this. ;-)
Quoted 3 lines
Not patching should at the very least cause a WARN with RETPOLINE kernels though, we hard rely on the patching actually working and writing at least a direct call.
We hard rely in other places too.
How about just BUG_ON()-ing in paravirt_patch_call() as well? It's not like these are *supposed* to fail, and if they do we want to know it, even if the outcome might be more benign in some cases?
I.e. how about the patch below? This not only makes it more apparent when patching fails, it also makes the kernel smaller and removes an #ifdef ugly.
I tried it with a richly paravirt-enabled kernel and no patching bugs were detected.
Thanks,
Ingo
--- arch/x86/kernel/paravirt.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
arch/x86/kernel/paravirt.c +8 -8
@@ -73,21 +73,21 @@ struct branch {static unsigned paravirt_patch_call (void *insnbuf,const void *target,unsigned long addr,unsigned len) { +const int call_len =5 ;struct branch *b = insnbuf; -unsigned long delta = (unsigned long )target - (addr+5 ); +unsigned long delta = (unsigned long )target - (addr+call_len); -if (len <5 ) { -#ifdef CONFIG_RETPOLINE -WARN_ONCE (1 ,"Failing to patch indirect CALL in %ps\n" , (void *)addr); -#endif -return len;/* call too long for patch site */ +if (len < call_len) { +pr_warn ("paravirt: Failed to patch indirect CALL at %ps\n" , (void *)addr); +/* Kernel might not be viable if patching fails, bail out: */ +BUG_ON (1 ); } b->opcode =0xe8 ;/* call */ b->delta = delta; -BUILD_BUG_ON (sizeof (*b) !=5 ); +BUILD_BUG_ON (sizeof (*b) != call_len); -return 5 ; +return call_len; }#ifdef CONFIG_PARAVIRT_XXL
Re: x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
Quoted 12 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
That's OK. The compiler emits an indirect CALL/JMP to the
pv_opsstructure contents. That _should_ stay valid and function correctly at all times.
Quoted 6 lines
It might result in a correctly executing kernel in terms of code generation, but it doesn't result in a viable kernel: some of the places rely on the patching going through and don't know what to do when it doesn't and misbehave or crash in interesting ways.
Guess how I know this. ;-)
What sites would that be? It really should work AFAIK.
Re: x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
That's OK. The compiler emits an indirect CALL/JMP to the
pv_opsstructure contents. That _should_ stay valid and function correctly at all times.
Quoted 8 lines
Quoted 6 lines
It might result in a correctly executing kernel in terms of code generation, but it doesn't result in a viable kernel: some of the places rely on the patching going through and don't know what to do when it doesn't and misbehave or crash in interesting ways.
Guess how I know this. ;-)
What sites would that be? It really should work AFAIK.
So for example I tried to increasing the size of one of the struct patch_xxl members:
arch/x86/kernel/paravirt_patch.c +1 -1
@@ -28,7 +28,7 @@ struct patch_xxl {const unsigned char irq_restore_fl[2 ];# ifdef CONFIG_X86_64 const unsigned char cpu_wbinvd[2 ]; -const unsigned char cpu_usergs_sysret64[6 ]; +const unsigned char cpu_usergs_sysret64[60 ];const unsigned char cpu_swapgs[3 ];const unsigned char mov64[3 ];# else
Which with the vanilla kernel crashes on boot much, much later:
[ 2.478026] PANIC: double fault, error_code: 0x0
But in any case, even if many of the others will work if the patching fails silently, is there any case where we'd treat patching failure as an acceptable case?
BUG_ON() in paravirt kernels is an easily debuggable condition and beats the above kinds of symptoms. But I can turn it into a WARN_ON_ONCE() if you think that's better?
Thanks,
Ingo
Juergen Gross
maintainerRe: x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
Quoted 40 lines · Ingo Molnar
Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
That's OK. The compiler emits an indirect CALL/JMP to the
pv_opsstructure contents. That _should_ stay valid and function correctly at all times.Quoted 8 lines
Quoted 6 lines
It might result in a correctly executing kernel in terms of code generation, but it doesn't result in a viable kernel: some of the places rely on the patching going through and don't know what to do when it doesn't and misbehave or crash in interesting ways.
Guess how I know this. ;-)
What sites would that be? It really should work AFAIK.
So for example I tried to increasing the size of one of the struct
patch_xxlmembers:arch/x86/kernel/paravirt_patch.c +1
-1@@ -28,7 +28,7 @@ struct patch_xxl {const unsigned char irq_restore_fl[2 ];# ifdef CONFIG_X86_64 const unsigned char cpu_wbinvd[2 ]; -const unsigned char cpu_usergs_sysret64[6 ]; +const unsigned char cpu_usergs_sysret64[60 ];const unsigned char cpu_swapgs[3 ];const unsigned char mov64[3 ];# else
Quoted 3 lines
Which with the vanilla kernel crashes on boot much, much later:
[ 2.478026] PANIC: double fault, error_code: 0x0
Sure, there is no NOP padding applied. Pre-populating the area with 1 byte NOPs would avoid the crash.
Quoted 7 lines
But in any case, even if many of the others will work if the patching fails silently, is there any case where we'd treat patching failure as an acceptable case?
BUG_ON()in paravirt kernels is an easily debuggable condition and beats the above kinds of symptoms. But I can turn it into aWARN_ON_ONCE()if you think that's better?
I'd prefer the BUG_ON(). Its not as if those conditions will occur on very few machines only. In case some patching isn't working we should catch those issues early.
Juergen
Juergen Gross
maintainerRe: x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
Quoted 41 lines · Juergen Gross
Quoted 40 lines · Ingo Molnar
Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
That's OK. The compiler emits an indirect CALL/JMP to the
pv_opsstructure contents. That _should_ stay valid and function correctly at all times.Quoted 8 lines
Quoted 6 lines
It might result in a correctly executing kernel in terms of code generation, but it doesn't result in a viable kernel: some of the places rely on the patching going through and don't know what to do when it doesn't and misbehave or crash in interesting ways.
Guess how I know this. ;-)
What sites would that be? It really should work AFAIK.
So for example I tried to increasing the size of one of the struct
patch_xxlmembers:arch/x86/kernel/paravirt_patch.c +1
-1@@ -28,7 +28,7 @@ struct patch_xxl {const unsigned char irq_restore_fl[2 ];# ifdef CONFIG_X86_64 const unsigned char cpu_wbinvd[2 ]; -const unsigned char cpu_usergs_sysret64[6 ]; +const unsigned char cpu_usergs_sysret64[60 ];const unsigned char cpu_swapgs[3 ];const unsigned char mov64[3 ];# else
Quoted 6 lines
Quoted 3 lines
Which with the vanilla kernel crashes on boot much, much later:
[ 2.478026] PANIC: double fault, error_code: 0x0Sure, there is no NOP padding applied. Pre-populating the area with 1 byte NOPs would avoid the crash.
This is wrong, of course.
But the indirect jmp is failing as struct pv_ops isn't mapped by the user page tables.
Juergen
Re: x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
In reply to Ingo MolnarQuoted 27 lines · Ingo Molnar
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
Quoted 3 lines · Ingo Molnar
It basically means that we silently won't do any patching and the kernel will crash later on in mysterious ways, because paravirt patching is usually relied on.
That's OK. The compiler emits an indirect CALL/JMP to the
pv_opsstructure contents. That _should_ stay valid and function correctly at all times.Quoted 8 lines
Quoted 6 lines
It might result in a correctly executing kernel in terms of code generation, but it doesn't result in a viable kernel: some of the places rely on the patching going through and don't know what to do when it doesn't and misbehave or crash in interesting ways.
Guess how I know this. ;-)
What sites would that be? It really should work AFAIK.
So for example I tried to increasing the size of one of the struct
patch_xxlmembers:
arch/x86/kernel/paravirt_patch.c struct patch_xxl { · line 34
Quoted 8 lines
arch/x86/kernel/paravirt_patch.c +1
-1@@ -28,7 +28,7 @@ struct patch_xxl {const unsigned char irq_restore_fl[2 ];# ifdef CONFIG_X86_64 const unsigned char cpu_wbinvd[2 ]; -const unsigned char cpu_usergs_sysret64[6 ]; +const unsigned char cpu_usergs_sysret64[60 ];const unsigned char cpu_swapgs[3 ];const unsigned char mov64[3 ];# else
So this then fails to patch the immediate; but the compiler emitted:
175: ff 25 00 00 00 00 jmpq *0x0(%rip) # 17b <syscall_return_via_sysret+0x75>
177: R_X86_64_PC32 pv_ops+0xfc
and pv_ops+0xfc is (+4 because of reloc magic):
void (*usergs_sysret64)(void); /* 0x100 0x8 */
which defaults to:
arch/x86/kernel/paravirt.c: .cpu.usergs_sysret64 = native_usergs_sysret64,
which in turn reads like:
0000000000000000 <native_usergs_sysret64>:
0: 0f 01 f8 swapgs 3: 48 0f 07 sysretq
So I _really_ don't understand how:
Quoted 3 lines
Which with the vanilla kernel crashes on boot much, much later:
[ 2.478026] PANIC: double fault, error_code: 0x0
happens.
Quoted 3 lines
But in any case, even if many of the others will work if the patching fails silently, is there any case where we'd treat patching failure as an acceptable case?
It really should just work. And we need to figure out why it comes unstuck. Can you print the code when it fails patching?
Quoted 3 lines
BUG_ON()in paravirt kernels is an easily debuggable condition and beats the above kinds of symptoms. But I can turn it into aWARN_ON_ONCE()if you think that's better?
Not patching should be a performance issue; not a correctness issue, as per the above. So WARN is the right thing.
Re: x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
Quoted 19 lines · Peter Zijlstra
So this then fails to patch the immediate; but the compiler emitted:
175: ff 25 00 00 00 00 jmpq *0x0(%rip) # 17b <
syscall_return_via_sysret+0x75>177: R_X86_64_PC32 pv_ops+0xfcand
pv_ops+0xfcis (+4 because of reloc magic):void (*usergs_sysret64)(void); /*
0x1000x8*/which defaults to:
arch/x86/kernel/paravirt.c: .cpu.usergs_sysret64 =native_usergs_sysret64,which in turn reads like:
0000000000000000 <
native_usergs_sysret64>:0: 0f 01 f8 swapgs 3: 48 0f 07 sysretq
Quoted 7 lines
So I _really_ don't understand how:
Quoted 3 lines
Which with the vanilla kernel crashes on boot much, much later:
[ 2.478026] PANIC: double fault, error_code: 0x0happens.
Just spoke to Juergen on IRC; the most likely explanation is that (because of PTI) the pv_ops table isn't mapped in the user page-tables and therefore the indirect jump comes unstuck.
tip-bot for Ingo Molnar
merge bot[tip:x86/paravirt] x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
In reply to Ingo Molnarx86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
Commit-ID: 11e86dc7f2746210f9c7dc10deaa7658f8dc8350 Gitweb: https://git.kernel.org/tip/11e86dc7f2746210f9c7dc10deaa7658f8dc8350 Author: Ingo Molnar <[email protected]> AuthorDate: Thu, 25 Apr 2019 11:50:39 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 12:00:44 +0200
x86/paravirt: Detect over-sized patching bugs in paravirt_patch_call()
paravirt_patch_call() currently handles patching failures inconsistently:
we generate a warning in the retpoline case, but don't in other cases where we might end up with a non-working kernel as well.
So just convert it all to a BUG_ON(), these patching calls are *not* supposed to fail, and if they do we want to know it immediately.
This also makes the kernel smaller and removes an #ifdef ugly.
I tried it with a richly paravirt-enabled kernel and no patching bugs were detected.
--- arch/x86/kernel/paravirt.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
arch/x86/kernel/paravirt.c +8 -8
@@ -73,21 +73,21 @@ struct branch {static unsigned paravirt_patch_call (void *insnbuf,const void *target,unsigned long addr,unsigned len) { +const int call_len =5 ;struct branch *b = insnbuf; -unsigned long delta = (unsigned long )target - (addr+5 ); +unsigned long delta = (unsigned long )target - (addr+call_len); -if (len <5 ) { -#ifdef CONFIG_RETPOLINE -WARN_ONCE (1 ,"Failing to patch indirect CALL in %ps\n" , (void *)addr); -#endif -return len;/* call too long for patch site */ +if (len < call_len) { +pr_warn ("paravirt: Failed to patch indirect CALL at %ps\n" , (void *)addr); +/* Kernel might not be viable if patching fails, bail out: */ +BUG_ON (1 ); } b->opcode =0xe8 ;/* call */ b->delta = delta; -BUILD_BUG_ON (sizeof (*b) !=5 ); +BUILD_BUG_ON (sizeof (*b) != call_len); -return 5 ; +return call_len; }#ifdef CONFIG_PARAVIRT_XXL
tip-bot for Ingo Molnar
merge bot[tip:x86/paravirt] x86/paravirt: Detect over-sized patching bugs in paravirt_patch_insns()
In reply to Ingo Molnarx86/paravirt: Detect over-sized patching bugs in paravirt_patch_insns()
Commit-ID: 2777cae2b19d4a08ad233b3504c19c6f7a6a2ef3 Gitweb: https://git.kernel.org/tip/2777cae2b19d4a08ad233b3504c19c6f7a6a2ef3 Author: Ingo Molnar <[email protected]> AuthorDate: Thu, 25 Apr 2019 11:17:17 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 12:00:31 +0200
x86/paravirt: Detect over-sized patching bugs in paravirt_patch_insns()
So paravirt_patch_insns() contains this gem of logic:
unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
const char *start, const char *end)
{
unsigned insn_len = end - start;
if (insn_len > len || start == NULL)
insn_len = len;
else
memcpy(insnbuf, start, insn_len);
return insn_len;
}Note how 'len' (size of the original instruction) is checked against the new instruction, and silently discarded with no warning printed whatsoever.
This crashes the kernel in funny ways if the patching template is buggy, and usually in much later places.
Instead do a direct BUG_ON(), there's no way to continue successfully at that point.
I've tested this patch, with the vanilla kernel check never triggers, and if I intentionally increase the size of one of the patch templates to a too high value the assert triggers:
[ 0.164385] kernel BUG at arch/x86/kernel/paravirt.c:167!
Without this patch a broken kernel randomly crashes in later places, after the silent patching failure.
--- arch/x86/kernel/paravirt.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
arch/x86/kernel/paravirt.c +4 -4
@@ -163,10 +163,10 @@ unsigned paravirt_patch_insns(void *insnbuf, unsigned len, {unsigned insn_len = end - start; -if (insn_len > len || start ==NULL ) - insn_len = len; -else -memcpy (insnbuf, start, insn_len); +/* Alternative instruction is too large for the patch site and we cannot continue: */ +BUG_ON (insn_len > len || start ==NULL ); + +memcpy (insnbuf, start, insn_len);return insn_len; }
tip-bot for Ingo Molnar
merge bot[tip:x86/paravirt] x86/paravirt: Match paravirt patchlet field definition ordering to initialization ordering
In reply to Ingo Molnarx86/paravirt: Match paravirt patchlet field definition ordering to initialization ordering
Commit-ID: fc93dfd9345bb8b29a62b21cb0447dd1a3815f91 Gitweb: https://git.kernel.org/tip/fc93dfd9345bb8b29a62b21cb0447dd1a3815f91 Author: Ingo Molnar <[email protected]> AuthorDate: Thu, 25 Apr 2019 10:10:12 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 12:00:44 +0200
x86/paravirt: Match paravirt patchlet field definition ordering to initialization ordering
Here's the objdump -D output of the PATCH_XXL data table:
0000000000000010 <patch_data_xxl>:
10: fa cli 11: fb sti 12: 57 push %rdi 13: 9d popfq 14: 9c pushfq 15: 58 pop %rax 16: 0f 20 d0 mov %cr2,%rax 19: 0f 20 d8 mov %cr3,%rax 1c: 0f 22 df mov %rdi,%cr3 1f: 0f 09 wbinvd 21: 0f 01 f8 swapgs 24: 48 0f 07 sysretq 27: 0f 01 f8 swapgs 2a: 48 89 f8 mov %rdi,%rax
Note how this doesn't match up to the source code:
static const struct patch_xxl patch_data_xxl = {
.irq_irq_disable = { 0xfa }, // cli
.irq_irq_enable = { 0xfb }, // sti
.irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax
.mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax
.mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax
.irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq
.mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3
.cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd
.cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8,
0x48, 0x0f, 0x07 }, // swapgs; sysretq
.cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs
.mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax
.irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf
.mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3
.cpu_iret = { 0xcf }, // iret
};Note how they are reordered: in the generated code .irq_restore_fl comes before .irq_save_fl, etc. This is because the field ordering in struct patch_xxl does not match the initialization ordering of patch_data_xxl.
Match up the initialization order with the definition order - this makes the disassembly easily reviewable:
0000000000000010 <patch_data_xxl>:
10: fa cli 11: fb sti 12: 9c pushfq 13: 58 pop %rax 14: 0f 20 d0 mov %cr2,%rax 17: 0f 20 d8 mov %cr3,%rax 1a: 0f 22 df mov %rdi,%cr3 1d: 57 push %rdi 1e: 9d popfq 1f: 0f 09 wbinvd 21: 0f 01 f8 swapgs 24: 48 0f 07 sysretq 27: 0f 01 f8 swapgs 2a: 48 89 f8 mov %rdi,%rax
--- arch/x86/kernel/paravirt_patch.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
arch/x86/kernel/paravirt_patch.c +3 -3
@@ -21,11 +21,11 @@struct patch_xxl {const unsigned char irq_irq_disable[1 ];const unsigned char irq_irq_enable[1 ]; -const unsigned char irq_restore_fl[2 ];const unsigned char irq_save_fl[2 ];const unsigned char mmu_read_cr2[3 ];const unsigned char mmu_read_cr3[3 ];const unsigned char mmu_write_cr3[3 ]; +const unsigned char irq_restore_fl[2 ];# ifdef CONFIG_X86_64 const unsigned char cpu_wbinvd[2 ];const unsigned char cpu_usergs_sysret64[6 ];
@@ -43,16 +43,16 @@ static const struct patch_xxl patch_data_xxl = { .mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax .mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax # ifdef CONFIG_X86_64 - .irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq .mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3 + .irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq .cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd .cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8, 0x48, 0x0f, 0x07 }, // swapgs; sysretq .cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs .mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax # else - .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf .cpu_iret = { 0xcf }, // iret # endif };
Re: [patch 3/3] x86/paravirt: Replace paravirt patch asm magic
In reply to Ingo MolnarQuoted 34 lines · Ingo Molnar
Quoted 19 lines
(quoted diff) +9
-9-# ifdef CONFIG_PARAVIRT_XXL -DEF_NATIVE(irq, irq_disable, "cli"); -DEF_NATIVE(irq, irq_enable, "sti"); -DEF_NATIVE(irq, restore_fl, "push %eax; popf"); -DEF_NATIVE(irq, save_fl, "pushf; pop %eax"); -DEF_NATIVE(cpu, iret, "iret"); -DEF_NATIVE(mmu, read_cr2, "mov %cr2, %eax"); -DEF_NATIVE(mmu, write_cr3, "mov %eax, %cr3"); -DEF_NATIVE(mmu, read_cr3, "mov %cr3, %eax"); +static const struct patch_xxl patch_data_xxl = { + .irq_irq_disable = { 0xfa }, // cli + .irq_irq_enable = { 0xfb }, // sti + .irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax + .mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax + .mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax +# ifdef CONFIG_X86_64 + .irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq + .mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3Review
Quoted 10 lines
(quoted diff) +10
-0+ .cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd + .cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8, + 0x48, 0x0f, 0x07 }, // swapgs; sysretq + .cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs + .mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax +# else + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf + .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .cpu_iret = { 0xcf }, // iret +# endifI think these open-coded hexa versions are somewhat fragile as well, how about putting these into a .S file and controlling the sections in an LTO safe manner there?
Quoted 2 lines
That will also allow us to write proper asm, and global labels can be used to extract the patchlets and their length?
While I'm not fan either; I think that will be worse still, because it splits the information over multiple files.
The advantage of this form is that it is clear how long the instructions are, which is important for the patching. These immediates have to be shorter than 5 bytes because they overwrite the CALL/JMP to the paravirt function.
/me eyes .cpu_usergs_sysret64 and goes wtf..
Re: [patch 3/3] x86/paravirt: Replace paravirt patch asm magic
Quoted 42 lines · Peter Zijlstra
Quoted 34 lines · Ingo Molnar
Quoted 19 lines
(quoted diff) +9
-9-# ifdef CONFIG_PARAVIRT_XXL -DEF_NATIVE(irq, irq_disable, "cli"); -DEF_NATIVE(irq, irq_enable, "sti"); -DEF_NATIVE(irq, restore_fl, "push %eax; popf"); -DEF_NATIVE(irq, save_fl, "pushf; pop %eax"); -DEF_NATIVE(cpu, iret, "iret"); -DEF_NATIVE(mmu, read_cr2, "mov %cr2, %eax"); -DEF_NATIVE(mmu, write_cr3, "mov %eax, %cr3"); -DEF_NATIVE(mmu, read_cr3, "mov %cr3, %eax"); +static const struct patch_xxl patch_data_xxl = { + .irq_irq_disable = { 0xfa }, // cli + .irq_irq_enable = { 0xfb }, // sti + .irq_save_fl = { 0x9c, 0x58 }, // pushf; pop %[re]ax + .mmu_read_cr2 = { 0x0f, 0x20, 0xd0 }, // mov %cr2, %[re]ax + .mmu_read_cr3 = { 0x0f, 0x20, 0xd8 }, // mov %cr3, %[re]ax +# ifdef CONFIG_X86_64 + .irq_restore_fl = { 0x57, 0x9d }, // push %rdi; popfq + .mmu_write_cr3 = { 0x0f, 0x22, 0xdf }, // mov %rdi, %cr3Review
Quoted 10 lines
(quoted diff) +10
-0+ .cpu_wbinvd = { 0x0f, 0x09 }, // wbinvd + .cpu_usergs_sysret64 = { 0x0f, 0x01, 0xf8, + 0x48, 0x0f, 0x07 }, // swapgs; sysretq + .cpu_swapgs = { 0x0f, 0x01, 0xf8 }, // swapgs + .mov64 = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax +# else + .irq_restore_fl = { 0x50, 0x9d }, // push %eax; popf + .mmu_write_cr3 = { 0x0f, 0x22, 0xd8 }, // mov %eax, %cr3 + .cpu_iret = { 0xcf }, // iret +# endifI think these open-coded hexa versions are somewhat fragile as well, how about putting these into a .S file and controlling the sections in an LTO safe manner there?
Quoted 2 lines
That will also allow us to write proper asm, and global labels can be used to extract the patchlets and their length?
While I'm not fan either; I think that will be worse still, because it splits the information over multiple files.
Quoted 6 lines
The advantage of this form is that it is clear how long the instructions are, which is important for the patching. These immediates have to be shorter than 5 bytes because they overwrite the CALL/JMP to the paravirt function.
/me eyes .cpu_usergs_sysret64 and goes wtf..
OK, my bad, the indirect CALL/JMP is 6 bytes.. phew.
Re: [patch 3/3] x86/paravirt: Replace paravirt patch asm magic
In reply to Peter ZijlstraQuoted 9 lines · Peter Zijlstra
Quoted 6 lines
I think these open-coded hexa versions are somewhat fragile as well, how about putting these into a .S file and controlling the sections in an LTO safe manner there?
That will also allow us to write proper asm, and global labels can be used to extract the patchlets and their length?
While I'm not fan either; I think that will be worse still, because it splits the information over multiple files.
Yeah, so that's a drawback of the .S files.
Quoted 6 lines
The advantage of this form is that it is clear how long the instructions are, which is important for the patching. These immediates have to be shorter than 5 bytes because they overwrite the CALL/JMP to the paravirt function.
/me eyes .cpu_usergs_sysret64 and goes wtf..
I just posted a patch that adds an assert to detect too large patching
attempt: we'd silently ignore them before, which isn't healthy.
With the two patches I now like the .c version better.
Thomas, want me to organize all these changes, or do you want to? 1
Thanks,
Ingo
tip-bot for Thomas Gleixner
merge bot[tip:x86/paravirt] x86/paravirt: Replace the paravirt patch asm magic
In reply to Thomas Gleixnerx86/paravirt: Replace the paravirt patch asm magic
Commit-ID: 0b9d2fc1d0d628c94c6866a2ed3005c6730db512 Gitweb: https://git.kernel.org/tip/0b9d2fc1d0d628c94c6866a2ed3005c6730db512 Author: Thomas Gleixner <[email protected]> AuthorDate: Wed, 24 Apr 2019 15:41:18 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 25 Apr 2019 12:00:44 +0200
x86/paravirt: Replace the paravirt patch asm magic
The magic macro DEF_NATIVE() in the paravirt patching code uses inline assembly to generate a data table for patching in the native instructions.
While clever this is falling apart with LTO and even aside of LTO the construct is just working by chance according to GCC folks.
Aside of that the tables are constant data and not some form of magic text.
As these constructs are not subject to frequent changes it is not a maintenance issue to convert them to regular data tables which are initialized with hex bytes.
Create a new set of macros and data structures to store the instruction sequences and convert the code over.
--- arch/x86/include/asm/paravirt_types.h | 4 - arch/x86/kernel/paravirt_patch.c | 142 +++++++++++++++++++--------------- 2 files changed, 81 insertions(+), 65 deletions(-)
arch/x86/include/asm/paravirt_types.h +0 -4
@@ -370,10 +370,6 @@ extern struct paravirt_patch_template pv_ops;/* Simple instruction patching code. */ #define NATIVE_LABEL (a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t" -#define DEF_NATIVE (ops, name, code) \ - __visible extern const char start_##ops##_##name[], end_##ops##_##name[]; \ - asm(NATIVE_LABEL("start_", ops, name) code NATIVE_LABEL("end_", ops, name)) -unsigned paravirt_patch_ident_64 (void *insnbuf,unsigned len);unsigned paravirt_patch_default (u8 type,void *insnbuf,unsigned long addr,unsigned len);
arch/x86/kernel/paravirt_patch.c +81 -61
184 lines not on this page · raw message