x86: Virtualize cr2
PATCH 1/1 x86: Virtualize cr2
Ensure that we always call the Linux page fault handler with the correct cr2 value set. So far this is not the case if a second fault happens while we execute the domain fault handler (which may, e.g., perform a sleepy migration before returning).
Instead of re-writing cr2 directly in hardware, make use of the paravirt capabilities of Linux and simply overload read/write_cr2 for this purpose.
--- arch/x86/include/asm/ipipe.h | 2 ++ arch/x86/include/asm/system.h | 5 +++++ arch/x86/kernel/ipipe.c | 10 ++++++++++ 3 files changed, 17 insertions(+), 0 deletions(-)
arch/x86/include/asm/ipipe.h +2 -0
@@ -33,6 +33,8 @@ DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs); +DECLARE_PER_CPU(unsigned long, __ipipe_cr2); +static inline unsigned __ipipe_get_irq_vector (int irq) {#ifdef CONFIG_X86_IO_APIC
arch/x86/include/asm/system.h +5 -0
@@ -308,8 +308,13 @@ static inline void native_wbinvd(void)#else #define read_cr0 () (native_read_cr0())#define write_cr0 (x) (native_write_cr0(x)) +#ifdef CONFIG_IPIPE +#define read_cr2 () __raw_get_cpu_var(__ipipe_cr2) +#define write_cr2 (x) __raw_get_cpu_var(__ipipe_cr2) = (x) +#else /* !CONFIG_IPIPE */ #define read_cr2 () (native_read_cr2())#define write_cr2 (x) (native_write_cr2(x)) +#endif /* !CONFIG_IPIPE */ #define read_cr3 () (native_read_cr3())#define write_cr3 (x) (native_write_cr3(x))#define read_cr4 () (native_read_cr4())
arch/x86/kernel/ipipe.c +10 -0
@@ -53,6 +53,9 @@ int __ipipe_tick_irq = 0; /* Legacy timer */ DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); +DEFINE_PER_CPU(unsigned long, __ipipe_cr2); +EXPORT_PER_CPU_SYMBOL_GPL (__ipipe_cr2); +#ifdef CONFIG_SMP static cpumask_t __ipipe_cpu_sync_map;
@@ -712,6 +715,7 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector) {bool root_entry = false;unsigned long flags =0 ; +unsigned long cr2 =0 ;if (ipipe_root_domain_p) { root_entry = true;
@@ -734,6 +738,9 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector)return 1 ;#endif /* CONFIG_KGDB */ +if (vector == ex_do_page_fault) + cr2 =native_read_cr2 (); +if (unlikely (ipipe_trap_notify (vector, regs))) {if (root_entry)local_irq_restore_nosync (flags);
@@ -779,6 +786,9 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector) } } +if (vector == ex_do_page_fault) +write_cr2 (cr2); + __ipipe_std_extable[vector](regs, error_code);/*
Quoted 8 lines · Jan Kiszka
Ensure that we always call the Linux page fault handler with the correct cr2 value set. So far this is not the case if a second fault happens while we execute the domain fault handler (which may, e.g., perform a sleepy migration before returning).
Instead of re-writing cr2 directly in hardware, make use of the paravirt capabilities of Linux and simply overload read/write_cr2 for this purpose.
Already had a chance to look at it? Any concerns? We are about to ship this patch as it proved to fix the bug we saw.
BTW: I haven't thought about all details yet, but this should finally allow us to enter the Linux page fault handler with hardware interrupts enabled, thus drop the related patches to enable them after reading cr2.
Jan
Quoted 83 lines
Signed-off-by Jan Kiszka <[email protected]>--- arch/x86/include/asm/ipipe.h | 2 ++ arch/x86/include/asm/system.h | 5 +++++ arch/x86/kernel/ipipe.c | 10 ++++++++++ 3 files changed, 17 insertions(+), 0 deletions(-)arch/x86/include/asm/ipipe.h +2
-0@@ -33,6 +33,8 @@ DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs); +DECLARE_PER_CPU(unsigned long, __ipipe_cr2); +static inline unsigned __ipipe_get_irq_vector (int irq) {#ifdef CONFIG_X86_IO_APIC arch/x86/include/asm/system.h +5
-0@@ -308,8 +308,13 @@ static inline void native_wbinvd(void)#else #define read_cr0 () (native_read_cr0())#define write_cr0 (x) (native_write_cr0(x)) +#ifdef CONFIG_IPIPE +#define read_cr2 () __raw_get_cpu_var(__ipipe_cr2) +#define write_cr2 (x) __raw_get_cpu_var(__ipipe_cr2) = (x) +#else /* !CONFIG_IPIPE */ #define read_cr2 () (native_read_cr2())#define write_cr2 (x) (native_write_cr2(x)) +#endif /* !CONFIG_IPIPE */ #define read_cr3 () (native_read_cr3())#define write_cr3 (x) (native_write_cr3(x))#define read_cr4 () (native_read_cr4())arch/x86/kernel/ipipe.c +10
-0@@ -53,6 +53,9 @@ int __ipipe_tick_irq = 0; /* Legacy timer */ DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); +DEFINE_PER_CPU(unsigned long, __ipipe_cr2); +EXPORT_PER_CPU_SYMBOL_GPL (__ipipe_cr2); +#ifdef CONFIG_SMP static cpumask_t __ipipe_cpu_sync_map;@@ -712,6 +715,7 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector) {bool root_entry = false;unsigned long flags =0 ; +unsigned long cr2 =0 ;if (ipipe_root_domain_p) { root_entry = true;@@ -734,6 +738,9 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector)return 1 ;#endif /* CONFIG_KGDB */ +if (vector == ex_do_page_fault) + cr2 =native_read_cr2 (); +if (unlikely (ipipe_trap_notify (vector, regs))) {if (root_entry)local_irq_restore_nosync (flags);@@ -779,6 +786,9 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector) } } +if (vector == ex_do_page_fault) +write_cr2 (cr2); + __ipipe_std_extable[vector](regs, error_code);/*
Signature · 2 lines
Siemens AG, Corporate Technology, CT T DE IT 1 Corporate Competence Center Embedded Linux
Quoted 6 lines · Jan Kiszka
Quoted 5 lines · Jan Kiszka
Ensure that we always call the Linux page fault handler with the correct cr2 value set. So far this is not the case if a second fault happens while we execute the domain fault handler (which may, e.g., perform a sleepy migration before returning).
Quoted 6 lines
Quoted 3 lines
Instead of re-writing cr2 directly in hardware, make use of the paravirt capabilities of Linux and simply overload read/write_cr2 for this purpose.
Already had a chance to look at it? Any concerns? We are about to ship this patch as it proved to fix the bug we saw.
No concerns, I'll merge this. Thanks.
Quoted 91 lines
BTW: I haven't thought about all details yet, but this should finally allow us to enter the Linux page fault handler with hardware interrupts enabled, thus drop the related patches to enable them after reading cr2.
Jan
Quoted 83 lines
Signed-off-by Jan Kiszka <[email protected]>--- arch/x86/include/asm/ipipe.h | 2 ++ arch/x86/include/asm/system.h | 5 +++++ arch/x86/kernel/ipipe.c | 10 ++++++++++ 3 files changed, 17 insertions(+), 0 deletions(-)arch/x86/include/asm/ipipe.h +2
-0@@ -33,6 +33,8 @@ DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs); +DECLARE_PER_CPU(unsigned long, __ipipe_cr2); +static inline unsigned __ipipe_get_irq_vector (int irq) {#ifdef CONFIG_X86_IO_APIC arch/x86/include/asm/system.h +5
-0@@ -308,8 +308,13 @@ static inline void native_wbinvd(void)#else #define read_cr0 () (native_read_cr0())#define write_cr0 (x) (native_write_cr0(x)) +#ifdef CONFIG_IPIPE +#define read_cr2 () __raw_get_cpu_var(__ipipe_cr2) +#define write_cr2 (x) __raw_get_cpu_var(__ipipe_cr2) = (x) +#else /* !CONFIG_IPIPE */ #define read_cr2 () (native_read_cr2())#define write_cr2 (x) (native_write_cr2(x)) +#endif /* !CONFIG_IPIPE */ #define read_cr3 () (native_read_cr3())#define write_cr3 (x) (native_write_cr3(x))#define read_cr4 () (native_read_cr4())arch/x86/kernel/ipipe.c +10
-0@@ -53,6 +53,9 @@ int __ipipe_tick_irq = 0; /* Legacy timer */ DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); +DEFINE_PER_CPU(unsigned long, __ipipe_cr2); +EXPORT_PER_CPU_SYMBOL_GPL (__ipipe_cr2); +#ifdef CONFIG_SMP static cpumask_t __ipipe_cpu_sync_map;@@ -712,6 +715,7 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector) {bool root_entry = false;unsigned long flags =0 ; +unsigned long cr2 =0 ;if (ipipe_root_domain_p) { root_entry = true;@@ -734,6 +738,9 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector)return 1 ;#endif /* CONFIG_KGDB */ +if (vector == ex_do_page_fault) + cr2 =native_read_cr2 (); +if (unlikely (ipipe_trap_notify (vector, regs))) {if (root_entry)local_irq_restore_nosync (flags);@@ -779,6 +786,9 @@ int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector) } } +if (vector == ex_do_page_fault) +write_cr2 (cr2); + __ipipe_std_extable[vector](regs, error_code);/*
Signature · 1 line
Philippe.