(Re-)introducing pvops for x86_64 - Real pvops work part
Subsystems (5)
cover (Re-)introducing pvops for x86_64 - Real pvops work part
Hey folks,
This is the part-of-pvops-implementation-that-is-not-exactly-a-merge. Neat, uh? This is the majority of the work.
The first patch in the series does not really belong here. It was already sent to lkml separetedly before, but I'm including it again, for a very simple reason: Try to test the paravirt patches without it, and you'll fail miserably ;-) (and it was not yet included).
Other than that, I thank you all in advance for the review.
Have fun!
PATCH 1/16 Wipe out traditional opt from x86_64 Makefile
Among other things, using -traditional as a gcc option stops us from using macro token pasting, which is a feature we heavily rely on.
There was still a use of -traditional in arch/x86/kernel/Makefile_64, which this patch removes.
I don't see any problems building kernels in my x86_64 box without -traditional.
--- arch/x86/kernel/Makefile_64 | 1 - 1 files changed, 0 insertions(+), 1 deletions(-)
arch/x86/kernel/Makefile_64 +0 -1
@@ -3,7 +3,6 @@ # extra-y := head_64.o head64.o init_task.o vmlinux.lds -EXTRA_AFLAGS := -traditional obj-y := process_64.o signal_64.o entry_64.o traps_64.o irq_64.o \ ptrace_64.o time_64.o ioport_64.o ldt_64.o setup_64.o i8259_64.o sys_x86_64.o \ x8664_ksyms_64.o i387_64.o syscall_64.o vsyscall_64.o \
Signature · 8 lines
1.4.4.2 ------------------------------------------------------------------------- This SF.net email is sponsored by: Splunk Inc. Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
PATCH 2/16 paravirt hooks at entry functions.
Those are the hooks needed for paravirt at entry_64.S In general, they follow the way of i386.
--- arch/x86/kernel/entry_64.S | 108 +++++++++++++++++++++++++++----------------- 1 files changed, 66 insertions(+), 42 deletions(-)
arch/x86/kernel/entry_64.S +66 -42
@@ -50,6 +50,7 @@ #include <asm/hw_irq.h> #include <asm/page.h> #include <asm/irqflags.h> +#include <asm/paravirt.h> .code64
@@ -57,6 +58,20 @@ #define retint_kernel retint_restore_args #endif +#ifdef CONFIG_PARAVIRT +ENTRY(native_irq_enable_syscall_ret) + movq %gs:pda_oldrsp,%rsp + swapgs + sysretq +/* + * This could well be defined as a C function, but as it is only used here, + * let it be locally defined + */ +ENTRY(native_swapgs) + swapgs + retq +#endif /* CONFIG_PARAVIRT */ + .macro TRACE_IRQS_IRETQ offset=ARGOFFSET #ifdef CONFIG_TRACE_IRQFLAGS
@@ -216,14 +231,21 @@ ENTRY(system_call) CFI_DEF_CFA rsp,PDA_STACKOFFSET CFI_REGISTER rip,rcx /*CFI_REGISTER rflags,r11*/ - swapgs + SWAPGS_UNSAFE_STACK + /* + * A hypervisor implementation might want to use a label + * after the swapgs, so that it can do the swapgs + * for the guest and jump here on syscall. + */ +ENTRY(system_call_after_swapgs) + movq %rsp,%gs:pda_oldrsp movq %gs:pda_kernelstack,%rsp /* * No need to follow this irqs off/on section - it's straight * and short: */ - sti + ENABLE_INTERRUPTS(CLBR_NONE) SAVE_ARGS 8,1 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp)
@@ -246,7 +268,7 @@ ret_from_sys_call: sysret_check: LOCKDEP_SYS_EXIT GET_THREAD_INFO(%rcx) - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF movl threadinfo_flags(%rcx),%edx andl %edi,%edx
@@ -260,9 +282,7 @@ sysret_check: CFI_REGISTER rip,rcx RESTORE_ARGS 0,-ARG_SKIP,1 /*CFI_REGISTER rflags,r11*/ - movq %gs:pda_oldrsp,%rsp - swapgs - sysretq + ENABLE_INTERRUPTS_SYSCALL_RET CFI_RESTORE_STATE /* Handle reschedules */
@@ -271,7 +291,7 @@ sysret_careful: bt $TIF_NEED_RESCHED,%edx jnc sysret_signal TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) pushq %rdi CFI_ADJUST_CFA_OFFSET 8 call schedule
@@ -282,7 +302,7 @@ sysret_careful: /* Handle a signal */ sysret_signal: TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) testl $(_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY),%edx jz 1f
@@ -295,7 +315,7 @@ sysret_signal: 1: movl $_TIF_NEED_RESCHED,%edi /* Use IRET because user could have changed frame. This works because ptregscall_common has called FIXUP_TOP_OF_STACK. */ - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp int_with_check
@@ -327,7 +347,7 @@ tracesys: */ .globl int_ret_from_sys_call int_ret_from_sys_call: - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF testl $3,CS-ARGOFFSET(%rsp) je retint_restore_args
@@ -349,20 +369,20 @@ int_careful: bt $TIF_NEED_RESCHED,%edx jnc int_very_careful TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) pushq %rdi CFI_ADJUST_CFA_OFFSET 8 call schedule popq %rdi CFI_ADJUST_CFA_OFFSET -8 - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp int_with_check /* handle signals and tracing -- both require a full stack frame */ int_very_careful: TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) SAVE_REST /* Check for syscall exit trace */ testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
@@ -385,7 +405,7 @@ int_signal: 1: movl $_TIF_NEED_RESCHED,%edi int_restore_rest: RESTORE_REST - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp int_with_check CFI_ENDPROC
@@ -506,7 +526,7 @@ END(stub_rt_sigreturn) CFI_DEF_CFA_REGISTER rbp testl $3,CS(%rdi) je 1f - swapgs + SWAPGS /* irqcount is used to check if a CPU is already on an interrupt stack or not. While this is essentially redundant with preempt_count it is a little cheaper to use a separate counter in the PDA
@@ -527,7 +547,7 @@ ENTRY(common_interrupt) interrupt do_IRQ /* 0(%rsp): oldrsp-ARGOFFSET */ ret_from_intr: - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF decl %gs:pda_irqcount leaveq
@@ -556,13 +576,13 @@ retint_swapgs: /* return to user-space */ /* * The iretq could re-enable interrupts: */ - cli + DISABLE_INTERRUPTS(CLBR_ANY) TRACE_IRQS_IRETQ - swapgs + SWAPGS jmp restore_args retint_restore_args: /* return to kernel space */ - cli + DISABLE_INTERRUPTS(CLBR_ANY) /* * The iretq could re-enable interrupts: */
@@ -570,10 +590,14 @@ retint_restore_args: /* return to kernel space */ restore_args: RESTORE_ARGS 0,8,0 iret_label: +#ifdef CONFIG_PARAVIRT + INTERRUPT_RETURN +#endif +ENTRY(native_iret) iretq .section __ex_table,"a" - .quad iret_label,bad_iret + .quad native_iret, bad_iret .previous .section .fixup,"ax" /* force a signal here? this matches i386 behaviour */
@@ -581,24 +605,24 @@ iret_label: bad_iret: movq $11,%rdi /* SIGSEGV */ TRACE_IRQS_ON - sti - jmp do_exit - .previous - + ENABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI)) + jmp do_exit + .previous + /* edi: workmask, edx: work */ retint_careful: CFI_RESTORE_STATE bt $TIF_NEED_RESCHED,%edx jnc retint_signal TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) pushq %rdi CFI_ADJUST_CFA_OFFSET 8 call schedule popq %rdi CFI_ADJUST_CFA_OFFSET -8 GET_THREAD_INFO(%rcx) - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp retint_check
@@ -606,14 +630,14 @@ retint_signal: testl $(_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY),%edx jz retint_swapgs TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) SAVE_REST movq $-1,ORIG_RAX(%rsp) xorl %esi,%esi # oldset movq %rsp,%rdi # &pt_regs call do_notify_resume RESTORE_REST - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF movl $_TIF_NEED_RESCHED,%edi GET_THREAD_INFO(%rcx)
@@ -731,7 +755,7 @@ END(spurious_interrupt) rdmsr testl %edx,%edx js 1f - swapgs + SWAPGS xorl %ebx,%ebx 1: .if \ist
@@ -747,7 +771,7 @@ END(spurious_interrupt) .if \ist addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) .endif - cli + DISABLE_INTERRUPTS(CLBR_NONE) .if \irqtrace TRACE_IRQS_OFF .endif
@@ -776,10 +800,10 @@ paranoid_swapgs\trace: .if \trace TRACE_IRQS_IRETQ 0 .endif - swapgs + SWAPGS_UNSAFE_STACK paranoid_restore\trace: RESTORE_ALL 8 - iretq + INTERRUPT_RETURN paranoid_userspace\trace: GET_THREAD_INFO(%rcx) movl threadinfo_flags(%rcx),%ebx
@@ -794,11 +818,11 @@ paranoid_userspace\trace: .if \trace TRACE_IRQS_ON .endif - sti + ENABLE_INTERRUPTS(CLBR_NONE) xorl %esi,%esi /* arg2: oldset */ movq %rsp,%rdi /* arg1: &pt_regs */ call do_notify_resume - cli + DISABLE_INTERRUPTS(CLBR_NONE) .if \trace TRACE_IRQS_OFF .endif
@@ -807,9 +831,9 @@ paranoid_schedule\trace: .if \trace TRACE_IRQS_ON .endif - sti + ENABLE_INTERRUPTS(CLBR_ANY) call schedule - cli + DISABLE_INTERRUPTS(CLBR_ANY) .if \trace TRACE_IRQS_OFF .endif
@@ -862,7 +886,7 @@ KPROBE_ENTRY(error_entry) testl $3,CS(%rsp) je error_kernelspace error_swapgs: - swapgs + SWAPGS error_sti: movq %rdi,RDI(%rsp) CFI_REL_OFFSET rdi,RDI
@@ -874,7 +898,7 @@ error_sti: error_exit: movl %ebx,%eax RESTORE_REST - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF GET_THREAD_INFO(%rcx) testl %eax,%eax
@@ -911,12 +935,12 @@ ENTRY(load_gs_index) CFI_STARTPROC pushf CFI_ADJUST_CFA_OFFSET 8 - cli - swapgs + DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI)) + SWAPGS gs_change: movl %edi,%gs 2: mfence /* workaround */ - swapgs + SWAPGS popf CFI_ADJUST_CFA_OFFSET -8 ret
@@ -930,7 +954,7 @@ ENDPROC(load_gs_index) .section .fixup,"ax" /* running with kernelgs */ bad_gs: - swapgs /* switch back to user gs */ + SWAPGS /* switch back to user gs */ xorl %eax,%eax movl %eax,%gs jmp 2b
Signature · 1 line
1.4.4.2
PATCH 3/16 read/write_crX, clts and wbinvd for 64-bit paravirt
This patch introduces, and patch callers when needed, native versions for read/write_crX functions, clts and wbinvd.
--- arch/x86/mm/pageattr_64.c | 3 +- include/asm-x86/system_64.h | 60 ++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 18 deletions(-)
arch/x86/mm/pageattr_64.c +2 -1
@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/tlbflush.h> #include <asm/io.h> +#include <asm/paravirt.h> pte_t *lookup_address (unsigned long address) {
@@ -77,7 +78,7 @@ static void flush_kernel_map(void *arg)much cheaper than WBINVD. */ /* clflush is still broken. Disable for now. */ if (1 || !cpu_has_clflush) - asmvolatile ("wbinvd" :::"memory" ); +wbinvd (); else list_for_each_entry(pg, l, lru) {void *adr =page_address (pg);clflush_cache_range (adr,PAGE_SIZE );
include/asm-x86/system_64.h +43 -17
106 lines not on this page · raw message
Signature · 1 line
1.4.4.2
PATCH 4/16 provide native irq initialization function
The interrupt initialization routine becomes native_init_IRQ and will be overriden later in case paravirt is on. The interrupt array is made visible for guests such lguest, that will need to have their own initialization mechanism (though using most of the same irq lines) later on.
--- arch/x86/kernel/i8259_64.c | 7 +++++-- include/asm-x86/irq_64.h | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/i8259_64.c +5 -2
@@ -76,7 +76,7 @@ BUILD_16_IRQS(0xc) BUILD_16_IRQS(0xd) BUILD_16_IRQS(0xe) BUILD_16_IRQS(0xf)IRQ (x,c),IRQ (x,d),IRQ (x,e),IRQ (x,f) /* for the irq vectors */ -static void (*interrupt[NR_VECTORS -FIRST_EXTERNAL_VECTOR ])(void ) = { +void (*interrupt[NR_VECTORS -FIRST_EXTERNAL_VECTOR ])(void ) = {IRQLIST_16 (0x2 ),IRQLIST_16 (0x3 ),IRQLIST_16 (0x4 ),IRQLIST_16 (0x5 ),IRQLIST_16 (0x6 ),IRQLIST_16 (0x7 ),IRQLIST_16 (0x8 ),IRQLIST_16 (0x9 ),IRQLIST_16 (0xa ),IRQLIST_16 (0xb ),
@@ -448,7 +448,10 @@ void __init init_ISA_irqs (void) } } -void __init init_IRQ (void ) +/* Overridden in paravirt.c */ +void init_IRQ (void ) __attribute__((weak,alias ("native_init_IRQ" ))); + +void __init native_init_IRQ (void ) {int i;
include/asm-x86/irq_64.h +3 -0
@@ -46,6 +46,9 @@ static __inline__ int irq_canonicalize(int irq)extern void fixup_irqs (cpumask_t map);#endif +#include <linux/init.h> +void native_init_IRQ (void ); +#define __ARCH_HAS_DO_SOFTIRQ 1#endif /* _ASM_IRQ_H */
Signature · 1 line
1.4.4.2
PATCH 5/16 report ring kernel is running without paravirt
When paravirtualization is disabled, the kernel is always running at ring 0. So report it in the appropriate macro
--- include/asm-x86/segment_64.h | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
include/asm-x86/segment_64.h +4 -0
@@ -50,4 +50,8 @@#define GDT_SIZE (GDT_ENTRIES * 8)#define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES * 8) +#ifndef CONFIG_PARAVIRT +#define get_kernel_rpl () 0 +#endif +#endif
Signature · 1 line
1.4.4.2
PATCH 6/16 export math_state_restore
Export math_state_restore symbol, so it can be used for hypervisors. They are commonly loaded as modules (lguest being an example).
--- arch/x86/kernel/traps_64.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
arch/x86/kernel/traps_64.c +1 -0
@@ -1077,6 +1077,7 @@ asmlinkage void math_state_restore(void)task_thread_info (me)->status |=TS_USEDFPU ; me->fpu_counter++; } +EXPORT_SYMBOL_GPL (math_state_restore);void __init trap_init (void ) {
Signature · 1 line
1.4.4.2
PATCH 7/16 native versions for set pagetables
This patch turns the set_p{te,md,ud,gd} functions into their native_ versions. There is no need to patch any caller.
Also, it adds pte_update() and pte_update_defer() calls whenever we modify a page table entry. This last part was coded to match i386 as close as possible.
Pieces of the header are moved to below the #ifdef CONFIG_PARAVIRT site, as they are users of the newly defined set_* macros.
--- include/asm-x86/pgtable_64.h | 192 ++++++++++++++++++++++++++++-------------- 1 files changed, 128 insertions(+), 64 deletions(-)
include/asm-x86/pgtable_64.h +128 -64
@@ -57,56 +57,107 @@ extern unsigned long empty_zero_page[PAGE_SIZE/sizeof(unsigned long)];*/ #define PTRS_PER_PTE 512 -#ifndef __ASSEMBLY__ +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else + +#define set_pte native_set_pte +#define set_pte_at (mm, addr, ptep, pteval) set_pte(ptep, pteval) +#define set_pmd native_set_pmd +#define set_pud native_set_pud +#define set_pgd native_set_pgd +#define pte_clear (mm, addr, xp) \ +do { \ + set_pte_at(mm, addr, xp, __pte(0)); \ +} while (0) -#define pte_ERROR (e) \ - printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), pte_val(e)) -#define pmd_ERROR (e) \ - printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), pmd_val(e)) -#define pud_ERROR (e) \ - printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), pud_val(e)) -#define pgd_ERROR (e) \ - printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e)) +#define pmd_clear (xp) do { set_pmd(xp, __pmd(0)); } while (0) +#define pud_clear native_pud_clear +#define pgd_clear native_pgd_clear +#define pte_update (mm, addr, ptep) do { } while (0) +#define pte_update_defer (mm, addr, ptep) do { } while (0) -#define pgd_none (x) (!pgd_val(x)) -#define pud_none (x) (!pud_val(x)) +#endif -static inline void set_pte (pte_t *dst,pte_t val) +#ifndef __ASSEMBLY__ + +static inline void native_set_pte (pte_t *dst,pte_t val) { -pte_val (*dst) =pte_val (val); + dst->pte =pte_val (val); } -#define set_pte_at (mm,addr,ptep,pteval) set_pte(ptep,pteval) -static inline void set_pmd (pmd_t *dst,pmd_t val) +static inline void native_set_pmd (pmd_t *dst,pmd_t val) { -pmd_val (*dst) =pmd_val (val); + dst->pmd =pmd_val (val); } -static inline void set_pud (pud_t *dst,pud_t val) +static inline void native_set_pud (pud_t *dst,pud_t val) { -pud_val (*dst) =pud_val (val); + dst->pud =pud_val (val); } -static inline void pud_clear (pud_t *pud) +static inline void native_set_pgd (pgd_t *dst,pgd_t val) { -set_pud (pud,__pud (0 )); + dst->pgd =pgd_val (val); } -static inline void set_pgd (pgd_t *dst,pgd_t val) +static inline void native_pud_clear (pud_t *pud) { -pgd_val (*dst) =pgd_val (val); -} +set_pud (pud,__pud (0 )); +} -static inline void pgd_clear (pgd_t * pgd) +static inline void native_pgd_clear (pgd_t *pgd) {set_pgd (pgd,__pgd (0 )); } -#define ptep_get_and_clear (mm,addr,xp) __pte(xchg(&(xp)->pte, 0)) +static inline void native_set_pte_at (struct mm_struct *mm,unsigned long addr, +pte_t *ptep,pte_t pteval) +{ +native_set_pte (ptep, pteval); +} + +static inline void native_pte_clear (struct mm_struct *mm,unsigned long addr, +pte_t *ptep) +{ +native_set_pte_at (mm, addr, ptep,__pte (0 )); +} + +static inline void native_pmd_clear (pmd_t *pmd) +{ +native_set_pmd (pmd,__pmd (0 )); +} + + +#define pte_ERROR (e) \ + printk("%s:%d: bad pte %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pte_val(e)) +#define pmd_ERROR (e) \ + printk("%s:%d: bad pmd %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pmd_val(e)) +#define pud_ERROR (e) \ + printk("%s:%d: bad pud %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pud_val(e)) +#define pgd_ERROR (e) \ + printk("%s:%d: bad pgd %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pgd_val(e)) + +#define pgd_none (x) (!pgd_val(x)) +#define pud_none (x) (!pud_val(x))struct mm_struct ; -static inline pte_t ptep_get_and_clear_full (struct mm_struct *mm,unsigned long addr,pte_t *ptep,int full) +static inline pte_t ptep_get_and_clear (struct mm_struct *mm, +unsigned long addr,pte_t *ptep) +{ +pte_t pte =__pte (xchg (&ptep->pte,0 )); +pte_update (mm, addr, ptep); +return pte; +} + +static inline pte_t ptep_get_and_clear_full (struct mm_struct *mm, +unsigned long addr,pte_t *ptep, +int full) {pte_t pte;if (full) {
@@ -246,7 +297,6 @@ static inline unsigned long pmd_bad(pmd_t pmd)#define pte_none (x) (!pte_val(x))#define pte_present (x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE)) -#define pte_clear (mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)#define pages_to_mb (x) ((x) >> (20-PAGE_SHIFT))/* FIXME: is this right? */
@@ -255,11 +305,11 @@ static inline unsigned long pmd_bad(pmd_t pmd)static inline pte_t pfn_pte (unsigned long page_nr,pgprot_t pgprot) { -pte_t pte; -pte_val (pte) = (page_nr <<PAGE_SHIFT ); -pte_val (pte) |=pgprot_val (pgprot); -pte_val (pte) &= __supported_pte_mask; -return pte; +unsigned long pte; + pte = (page_nr <<PAGE_SHIFT ); + pte |=pgprot_val (pgprot); + pte &= __supported_pte_mask; +return __pte (pte); }/*
@@ -283,30 +333,6 @@ static inline pte_t pte_mkwrite(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) |static inline pte_t pte_mkhuge (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_PSE));return pte; }static inline pte_t pte_clrhuge (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) & ~_PAGE_PSE));return pte; } -struct vm_area_struct ; - -static inline int ptep_test_and_clear_young (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep) -{ -if (!pte_young (*ptep)) -return 0 ; -return test_and_clear_bit (_PAGE_BIT_ACCESSED, &ptep->pte); -} - -static inline void ptep_set_wrprotect (struct mm_struct *mm,unsigned long addr,pte_t *ptep) -{ -clear_bit (_PAGE_BIT_RW, &ptep->pte); -} - -/* -* Macro to mark a page protection value as "uncacheable". -*/ -#define pgprot_noncached (prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) - -static inline int pmd_large (pmd_t pte) { -return (pmd_val (pte) & __LARGE_PTE) == __LARGE_PTE; -} - -/* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to.
@@ -340,7 +366,6 @@ static inline int pmd_large(pmd_t pte) { pmd_index(address))#define pmd_none (x) (!pmd_val(x))#define pmd_present (x) (pmd_val(x) & _PAGE_PRESENT) -#define pmd_clear (xp) do { set_pmd(xp, __pmd(0)); } while (0)#define pfn_pmd (nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot)))#define pmd_pfn (x) ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT)
@@ -352,15 +377,53 @@ static inline int pmd_large(pmd_t pte) {/* page, protection -> pte */ #define mk_pte (page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) -#define mk_pte_huge (entry) (pte_val(entry) |= _PAGE_PRESENT | _PAGE_PSE) - + +static inline pte_t __mk_pte_huge (pte_t entry) +{ +unsigned long pte; + pte =pte_val (entry); + pte |= _PAGE_PRESENT | _PAGE_PSE; +return __pte (pte); +} +#define mk_pte_huge (entry) ((entry) = __mk_pte_huge(entry)) + +#include <linux/mm_types.h> +static inline int ptep_test_and_clear_young (struct vm_area_struct *vma, +unsigned long addr,pte_t *ptep) +{ +int ret =0 ; +if (!pte_young (*ptep)) +return 0 ; + ret =test_and_clear_bit (_PAGE_BIT_ACCESSED, &ptep->pte); +pte_update (vma->vm_mm, addr, ptep); +return ret; +} + +static inline void ptep_set_wrprotect (struct mm_struct *mm, +unsigned long addr,pte_t *ptep) +{ +clear_bit (_PAGE_BIT_RW, &ptep->pte); +pte_update (mm, addr, ptep); +} + +/* +* Macro to mark a page protection value as "uncacheable". +*/ +#define pgprot_noncached (prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) + +static inline int pmd_large (pmd_t pte) +{ +return (pmd_val (pte) & __LARGE_PTE) == __LARGE_PTE; +} +/* Change flags of a PTE */ -static inline pte_t pte_modify (pte_t pte,pgprot_t newprot) +static inline pte_t pte_modify (pte_t pte_old,pgprot_t newprot) { -pte_val (pte) &= _PAGE_CHG_MASK; -pte_val (pte) |=pgprot_val (newprot); -pte_val (pte) &= __supported_pte_mask; -return pte; +unsigned long pte =pte_val (pte_old); + pte &= _PAGE_CHG_MASK; + pte |=pgprot_val (newprot); + pte &= __supported_pte_mask; +return __pte (pte); }#define pte_index (address) \
@@ -387,6 +450,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)int __changed = !pte_same (*(__ptep), __entry); \if (__changed && __dirty) { \set_pte (__ptep, __entry); \ +pte_update_defer ((__vma)->vm_mm, (__address), (__ptep)); \flush_tlb_page (__vma, __address); \ } \ __changed; \
Signature · 1 line
1.4.4.2
PATCH 8/16 add native functions for descriptors handling
This patch turns the basic descriptor handling into native_ functions. It is basically write_idt, load_idt, write_gdt, load_gdt, set_ldt, store_tr, load_tls, and the ones for updating a single entry.
In the process of doing that, we change the definition of load_LDT_nolock, and caller sites have to be patched. We also patch call sites that now needs a typecast.
--- arch/x86/kernel/ldt_64.c | 7 +- include/asm-x86/desc_64.h | 172 +++++++++++++++++++++++++------------- include/asm-x86/mmu_context_64.h | 23 ++++-- 3 files changed, 132 insertions(+), 70 deletions(-)
arch/x86/kernel/ldt_64.c +3 -4
@@ -171,7 +171,7 @@ static int write_ldt(void __user * ptr, unsigned long bytecount, int oldmode) {struct task_struct *me = current;struct mm_struct * mm = me->mm; -__u32 entry_1, entry_2, *lp; +__u32 entry_1, entry_2;int error;struct user_desc ldt_info;
@@ -200,7 +200,6 @@ static int write_ldt(void __user * ptr, unsigned long bytecount, int oldmode) goto out_unlock; } - lp = (__u32 *) ((ldt_info.entry_number <<3 ) + (char *) mm->context.ldt);/* Allow LDTs to be cleared by the user. */ if (ldt_info.base_addr ==0 && ldt_info.limit ==0 ) {
@@ -218,8 +217,8 @@ static int write_ldt(void __user * ptr, unsigned long bytecount, int oldmode)/* Install the new entry ... */ install: - *lp = entry_1; - *(lp+1 ) = entry_2; +write_ldt_entry (mm->context.ldt, ldt_info.entry_number, + entry_1, entry_2); error =0 ; out_unlock:
include/asm-x86/desc_64.h +112 -60
236 lines not on this page · raw message
include/asm-x86/mmu_context_64.h +17 -6
@@ -7,7 +7,16 @@#include <asm/pda.h> #include <asm/pgtable.h> #include <asm/tlbflush.h> + +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #include <asm-generic/mm_hooks.h> +static inline void paravirt_activate_mm (struct mm_struct *prev, +struct mm_struct *next) +{ +} +#endif /* CONFIG_PARAVIRT */ /* * possibly do the LDT unload here?
@@ -25,7 +34,7 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)static inline void load_cr3 (pgd_t *pgd) { - asmvolatile ("movq %0,%%cr3" ::"r" (__pa (pgd)) :"memory" ); +write_cr3 (__pa (pgd)); } static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
@@ -43,7 +52,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,load_cr3 (next->pgd);if (unlikely (next->context.ldt != prev->context.ldt)) -load_LDT_nolock (&next->context, cpu); +load_LDT_nolock (&next->context); }#ifdef CONFIG_SMP else {
@@ -56,7 +65,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,* to make sure to use no freed page tables. */ load_cr3 (next->pgd); - load_LDT_nolock(&next->context, cpu); + load_LDT_nolock(&next->context); } }#endif
@@ -67,8 +76,10 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, asmvolatile ("movl %0,%%fs" ::"r" (0 )); \ } while(0) -#define activate_mm (prev, next) \ - switch_mm((prev),(next),NULL) - +#define activate_mm (prev, next) \ +do { \ + paravirt_activate_mm(prev, next); \ + switch_mm((prev), (next), NULL); \ +} while (0)#endif
Signature · 1 line
1.4.4.2
PATCH 9/16 This patch add provisions for time related functions so they
can be later replaced by paravirt versions.
it basically encloses {g,s}et_wallclock inside the already existent functions update_persistent_clock and read_persistent_clock, and defines {s,g}et_wallclock to the core of such functions.
it also allow for a later-on-game time initialization, as done by i386. Paravirt guests can set a function to do their own initialization this way.
--- arch/x86/kernel/time_64.c | 42 ++++++++++++++++++++++++++++-------------- include/asm-x86/time.h | 26 ++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 18 deletions(-)
arch/x86/kernel/time_64.c +28 -14
@@ -33,9 +33,12 @@#include <acpi/acpi_bus.h> #endif #include <asm/i8253.h> +#include <asm/arch_hooks.h> #include <asm/pgtable.h> #include <asm/vsyscall.h> +#include <asm/time.h> #include <asm/timex.h> +#include <asm/timer.h> #include <asm/proto.h> #include <asm/hpet.h> #include <asm/sections.h>
@@ -77,18 +80,12 @@ EXPORT_SYMBOL(profile_pc);* sheet for details. */ -static int set_rtc_mmss (unsigned long nowtime) +int do_set_rtc_mmss (unsigned long nowtime) {int retval =0 ;int real_seconds, real_minutes, cmos_minutes;unsigned char control, freq_select; -/* -* IRQs are disabled when we're called from the timer interrupt, -* no need for spin_lock_irqsave() -*/ - -spin_lock (&rtc_lock);/* * Tell the clock it's being set and stop it.
@@ -138,14 +135,22 @@ static int set_rtc_mmss(unsigned long nowtime)CMOS_WRITE (control,RTC_CONTROL );CMOS_WRITE (freq_select,RTC_FREQ_SELECT ); -spin_unlock (&rtc_lock); -return retval; }int update_persistent_clock (struct timespec now) { -return set_rtc_mmss (now.tv_sec); +int retval; + +/* +* IRQs are disabled when we're called from the timer interrupt, +* no need for spin_lock_irqsave() +*/ +spin_lock (&rtc_lock); + retval =set_wallclock (now.tv_sec); +spin_unlock (&rtc_lock); + +return retval; }static irqreturn_t timer_event_interrupt (int irq,void *dev_id)
@@ -157,7 +162,7 @@ static irqreturn_t timer_event_interrupt(int irq, void *dev_id)return IRQ_HANDLED ; } -unsigned long read_persistent_clock (void ) +unsigned long do_get_cmos_time (void ) {unsigned int year, mon, day, hour, min, sec;unsigned long flags;
@@ -208,10 +213,15 @@ unsigned long read_persistent_clock(void)return mktime (year, mon, day, hour, min, sec); } +unsigned long read_persistent_clock (void ) +{ +return get_wallclock (); +} +/* calibrate_cpu is used on systems with fixed rate TSCs to determine * processor frequency */ #define TICK_COUNT 100000000 -static unsigned int __init tsc_calibrate_cpu_khz (void ) +unsigned long __init native_calculate_cpu_khz (void ) {int tsc_start, tsc_now;int i, no_ctr_free;
@@ -261,20 +271,23 @@ static struct irqaction irq0 = { .name = "timer" }; -void __init time_init (void ) +void __init hpet_time_init (void ) {if (!hpet_enable ())setup_pit_timer ();setup_irq (0 , &irq0); +} +void __init time_init (void ) +{tsc_calibrate (); cpu_khz = tsc_khz;if (cpu_has (&boot_cpu_data,X86_FEATURE_CONSTANT_TSC ) && boot_cpu_data.x86_vendor ==X86_VENDOR_AMD && boot_cpu_data.x86 ==16 ) - cpu_khz =tsc_calibrate_cpu_khz (); + cpu_khz =calculate_cpu_khz ();if (unsynchronized_tsc ())mark_tsc_unstable ("TSCs unsynchronized" );
@@ -287,4 +300,5 @@ void __init time_init(void)printk (KERN_INFO "time.c: Detected %d.%03d MHz processor.\n" , cpu_khz /1000 , cpu_khz %1000 );init_tsc_clocksource (); + late_time_init =choose_time_init (); }
include/asm-x86/time.h +22 -4
@@ -1,6 +1,10 @@ -#ifndef _ASMi386_TIME_H -#define _ASMi386_TIME_H +#ifndef _ASMX86_TIME_H +#define _ASMX86_TIME_H +extern void (*late_time_init)(void ); +extern void hpet_time_init (void ); + +#ifndef CONFIG_X86_64 #include <linux/efi.h> #include "mach_time.h"
@@ -28,8 +32,22 @@ static inline int native_set_wallclock(unsigned long nowtime) return retval; } -extern void (*late_time_init)(void ); -extern void hpet_time_init (void ); +#else +extern unsigned long do_get_cmos_time (void ); +extern int do_set_rtc_mmss (unsigned long nowtime); +extern void native_time_init_hook (void ); + +static inline unsigned long native_get_wallclock (void ) +{ +return do_get_cmos_time (); +} + +static inline int native_set_wallclock (unsigned long nowtime) +{ +return do_set_rtc_mmss (nowtime); +} + +#endif #ifdef CONFIG_PARAVIRT #include <asm/paravirt.h>
Signature · 8 lines
1.4.4.2 ------------------------------------------------------------------------- This SF.net email is sponsored by: Splunk Inc. Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
PATCH 10/16 export cpu_gdt_descr
With paravirualization, hypervisors needs to handle the gdt, that was right to this point only used at very early inialization code. Hypervisors (lguest being the current case) are commonly modules, so make it an export
--- arch/x86/kernel/x8664_ksyms_64.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-)
arch/x86/kernel/x8664_ksyms_64.c +6 -0
@@ -8,6 +8,7 @@#include <asm/processor.h> #include <asm/uaccess.h> #include <asm/pgtable.h> +#include <asm/desc.h> EXPORT_SYMBOL (kernel_thread);
@@ -60,3 +61,8 @@ EXPORT_SYMBOL(init_level4_pgt);EXPORT_SYMBOL (load_gs_index);EXPORT_SYMBOL (_proxy_pda); + +#ifdef CONFIG_PARAVIRT +/* Virtualized guests may want to use it */ +EXPORT_SYMBOL_GPL (cpu_gdt_descr); +#endif
Signature · 1 line
1.4.4.2
PATCH 11/16 turn priviled operation into a macro in head_64.S
under paravirt, read cr2 cannot be issued directly anymore. So wrap it in a macro, defined to the operation itself in case paravirt is off, but to something else if we have paravirt in the game
--- arch/x86/kernel/head_64.S | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-)
arch/x86/kernel/head_64.S +8 -1
@@ -19,6 +19,13 @@ #include <asm/msr.h> #include <asm/cache.h> +#ifdef CONFIG_PARAVIRT +#include <asm/asm-offsets.h> +#include <asm/paravirt.h> +#else +#define GET_CR2_INTO_RCX movq %cr2, %rcx +#endif + /* we are not able to switch in one step to the final KERNEL ADRESS SPACE * because we need identity-mapped pages. *
@@ -267,7 +274,7 @@ ENTRY(early_idt_handler) xorl %eax,%eax movq 8(%rsp),%rsi # get rip movq (%rsp),%rdx - movq %cr2,%rcx + GET_CR2_INTO_RCX leaq early_idt_msg(%rip),%rdi call early_printk cmpl $2,early_recursion_flag(%rip)
Signature · 1 line
1.4.4.2
PATCH 12/16 tweak io_64.h for paravirt.
We need something here because we can't call in and out instructions directly. However, we have to be careful, because no indirections are allowed in misc_64.c , and paravirt_ops is a kind of one. So just call it directly there
--- arch/x86/boot/compressed/misc_64.c | 6 +++++ include/asm-x86/io_64.h | 37 +++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-)
arch/x86/boot/compressed/misc_64.c +6 -0
@@ -9,6 +9,12 @@* High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996 */ +/* +* we have to be careful, because no indirections are allowed here, and +* paravirt_ops is a kind of one. As it will only run in baremetal anyway, +* we just keep it from happening +*/ +#undef CONFIG_PARAVIRT#define _LINUX_STRING_H_ 1#define __LINUX_BITMAP_H 1
include/asm-x86/io_64.h +30 -7
@@ -35,12 +35,24 @@* - Arnaldo Carvalho de Melo <acme-KCZ47A4bww4P48s/[email protected]> */ -#define __SLOW_DOWN_IO "\noutb %%al,$0x80" +static inline void native_io_delay (void ) +{ + asmvolatile ("outb %%al,$0x80" : : :"memory" ); +} -#ifdef REALLY_SLOW_IO -#define __FULL_SLOW_DOWN_IO __SLOW_DOWN_IO __SLOW_DOWN_IO __SLOW_DOWN_IO __SLOW_DOWN_IO +#if defined(CONFIG_PARAVIRT ) +#include <asm/paravirt.h> #else -#define __FULL_SLOW_DOWN_IO __SLOW_DOWN_IO + +static inline void slow_down_io (void ) +{ +native_io_delay (); +#ifdef REALLY_SLOW_IO +native_io_delay (); +native_io_delay (); +native_io_delay (); +#endif +}#endif /*
@@ -52,9 +64,15 @@ static inline void out##s(unsigned x value, unsigned short port) {#define __OUT2 (s,s1,s2) \ __asm__ __volatile__ ("out" #s " %" s1 "0,%" s2 "1" +#ifndef REALLY_SLOW_IO +#define REALLY_SLOW_IO +#define UNSET_REALLY_SLOW_IO +#endif +#define __OUT (s,s1,x) \ __OUT1(s,x) __OUT2(s,s1,"w") : : "a" (value), "Nd" (port)); } \ -__OUT1(s##_p,x) __OUT2(s,s1,"w") __FULL_SLOW_DOWN_IO : : "a" (value), "Nd" (port));} \ +__OUT1(s##_p, x) __OUT2(s, s1, "w") : : "a" (value), "Nd" (port)); \ + slow_down_io(); }#define __IN1 (s) \ static inline RETURN_TYPE in##s(unsigned short port) { RETURN_TYPE _v;
@@ -63,8 +81,13 @@ static inline RETURN_TYPE in##s(unsigned short port) { RETURN_TYPE _v; __asm__ __volatile__ ("in" #s " %" s2 "1,%" s1 "0" #define __IN(s,s1,i...) \ -__IN1(s) __IN2(s,s1,"w") : "=a" (_v) : "Nd" (port) ,##i ); return _v; } \ -__IN1(s##_p) __IN2(s,s1,"w") __FULL_SLOW_DOWN_IO : "=a" (_v) : "Nd" (port) ,##i ); return _v; } \ +__IN1(s) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); return _v; } \ +__IN1(s##_p) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); \ + slow_down_io(); return _v; } + +#ifdef UNSET_REALLY_SLOW_IO +#undef REALLY_SLOW_IO +#endif #define __INS (s) \ static inline void ins##s(unsigned short port, void * addr, unsigned long count) \
Signature · 8 lines
1.4.4.2 ------------------------------------------------------------------------- This SF.net email is sponsored by: Splunk Inc. Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
PATCH 13/16 native versions for page table entries values
This patch turns the page operations (set and make a page table) into native_ versions. The operations itself will be later overriden by paravirt.
It uses unsigned long long for consistency with 32-bit. So we have to fix fault_64.c to get rid of warnings.
--- arch/x86/mm/fault_64.c | 8 +++--- include/asm-x86/page_64.h | 56 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 9 deletions(-)
arch/x86/mm/fault_64.c +4 -4
@@ -158,22 +158,22 @@ void dump_pagetable(unsigned long address) pgd =__va ((unsigned long )pgd &PHYSICAL_PAGE_MASK ); pgd +=pgd_index (address);if (bad_address (pgd)) goto bad; -printk ("PGD %lx " ,pgd_val (*pgd)); +printk ("PGD %llx " ,pgd_val (*pgd));if (!pgd_present (*pgd)) goto ret; pud =pud_offset (pgd, address);if (bad_address (pud)) goto bad; -printk ("PUD %lx " ,pud_val (*pud)); +printk ("PUD %llx " ,pud_val (*pud));if (!pud_present (*pud)) goto ret; pmd =pmd_offset (pud, address);if (bad_address (pmd)) goto bad; -printk ("PMD %lx " ,pmd_val (*pmd)); +printk ("PMD %llx " ,pmd_val (*pmd));if (!pmd_present (*pmd) ||pmd_large (*pmd)) goto ret; pte =pte_offset_kernel (pmd, address);if (bad_address (pte)) goto bad; -printk ("PTE %lx" ,pte_val (*pte)); +printk ("PTE %llx" ,pte_val (*pte)); ret:printk ("\n" );return ;
include/asm-x86/page_64.h +51 -5
67 lines not on this page · raw message
Signature · 1 line
1.4.4.2
PATCH 14/16 prepare x86_64 architecture initialization for paravirt
This patch prepares the x86_64 architecture initialization for paravirt. It requires a memory initialization step, which is done by implementing 64-bit version for machine_specific_memory_setup, and putting an ARCH_SETUP hook, for guest-dependent initialization. This last step is done akin to i386
--- arch/x86/kernel/e820_64.c | 9 +++++++-- arch/x86/kernel/setup_64.c | 28 +++++++++++++++++++++++++++- include/asm-x86/setup.h | 11 ++++++++--- 3 files changed, 42 insertions(+), 6 deletions(-)
arch/x86/kernel/e820_64.c +7 -2
@@ -593,8 +593,10 @@ void early_panic(char *msg)panic (msg); } -void __init setup_memory_region (void ) +/* We're not void only for x86 32-bit compat */ +char *__init machine_specific_memory_setup (void ) { +char *who ="BIOS-e820" ;/* * Try to copy the BIOS-supplied E820-map. *
@@ -605,7 +607,10 @@ void __init setup_memory_region(void)if (copy_e820_map (boot_params.e820_map, boot_params.e820_entries) <0 )early_panic ("Cannot find a valid memory map" );printk (KERN_INFO "BIOS-provided physical RAM map:\n" ); -e820_print_map ("BIOS-e820" ); +e820_print_map (who); + +/* In case someone cares... */ +return who; }static int __init parse_memopt (char *p)
arch/x86/kernel/setup_64.c +27 -1
@@ -39,6 +39,7 @@#include <linux/dmi.h> #include <linux/dma-mapping.h> #include <linux/ctype.h> +#include <linux/uaccess.h> #include <asm/mtrr.h> #include <asm/uaccess.h>
@@ -60,6 +61,12 @@#include <asm/dmi.h> #include <asm/cacheflush.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define ARCH_SETUP +#endif +/* * Machine setup.. */
@@ -241,6 +248,16 @@ static void discover_ebda(void)* 4K EBDA area at 0x40E */ ebda_addr = *(unsigned short *)__va (EBDA_ADDR_POINTER ); +/* +* There can be some situations, like paravirtualized guests, +* in which there is no available ebda information. In such +* case, just skip it +*/ +if (!ebda_addr) { + ebda_size =0 ; +return ; + } + ebda_addr <<=4 ; ebda_size = *(unsigned short *)__va (ebda_addr);
@@ -254,6 +271,12 @@ static void discover_ebda(void) ebda_size =64 *1024 ; } +/* Overridden in paravirt.c if CONFIG_PARAVIRT */ +void __attribute__((weak))memory_setup (void ) +{ +machine_specific_memory_setup (); +} +void __init setup_arch (char **cmdline_p) {printk (KERN_INFO "Command line: %s\n" , boot_command_line);
@@ -269,7 +292,10 @@ void __init setup_arch(char **cmdline_p) rd_prompt = ((boot_params.hdr.ram_size &RAMDISK_PROMPT_FLAG ) !=0 ); rd_doload = ((boot_params.hdr.ram_size &RAMDISK_LOAD_FLAG ) !=0 );#endif -setup_memory_region (); + +ARCH_SETUP + +memory_setup ();copy_edd ();if (!boot_params.hdr.root_flags)
include/asm-x86/setup.h +8 -3
@@ -3,6 +3,13 @@#define COMMAND_LINE_SIZE 2048 +#ifndef __ASSEMBLY__ +char *machine_specific_memory_setup (void ); +#ifndef CONFIG_PARAVIRT +#define paravirt_post_allocator_init () do {} while (0) +#endif +#endif /* __ASSEMBLY__ */ +#ifdef __KERNEL__#ifdef __i386__
@@ -51,9 +58,7 @@ void __init add_memory_region(unsigned long long start,extern unsigned long init_pg_tables_end; -#ifndef CONFIG_PARAVIRT -#define paravirt_post_allocator_init () do {} while (0) -#endif +#endif /* __i386__ */ #endif /* _SETUP */
Signature · 1 line
1.4.4.2
PATCH 15/16 consolidation of paravirt for 32 and 64 bits
This patch unifies the paravirt ops structures for usage in x86_64 and i386 architectures. Some new field had to be created to accomodate the differences between the architectures.
--- arch/x86/Kconfig.x86_64 | 11 + arch/x86/kernel/Makefile | 3 + arch/x86/kernel/Makefile_32 | 1 - arch/x86/kernel/asm-offsets.c | 8 + arch/x86/kernel/asm-offsets_32.c | 8 - arch/x86/kernel/asm-offsets_64.c | 22 +- arch/x86/kernel/paravirt.c | 445 +++++++++++++++++++++++++++++++++ arch/x86/kernel/paravirt_32.c | 472 ----------------------------------- arch/x86/kernel/paravirt_patch_32.c | 52 ++++ arch/x86/kernel/paravirt_patch_64.c | 56 ++++ arch/x86/kernel/vmlinux_64.lds.S | 6 + include/asm-x86/paravirt.h | 458 ++++++++++++++++++++++++++++------ 12 files changed, 971 insertions(+), 571 deletions(-)
arch/x86/Kconfig.x86_64 +11 -0
@@ -372,6 +372,17 @@ config NODES_SHIFT # Dummy CONFIG option to select ACPI_NUMA from drivers/acpi/Kconfig. +config PARAVIRT + bool + depends on EXPERIMENTAL + help + Paravirtualization is a way of running multiple instances of + Linux on the same machine, under a hypervisor. This option + changes the kernel so it can modify itself when it is run + under a hypervisor, improving performance significantly. + However, when run without a hypervisor the kernel is + theoretically slower. If in doubt, say N. + config X86_64_ACPI_NUMA bool "ACPI NUMA detection" depends on NUMA
arch/x86/kernel/Makefile +3 -0
@@ -1,8 +1,11 @@ ifeq ($(CONFIG_X86_32),y) include ${srctree}/arch/x86/kernel/Makefile_32 +obj-$(CONFIG_PARAVIRT) += paravirt_patch_32.o else include ${srctree}/arch/x86/kernel/Makefile_64 +obj-$(CONFIG_PARAVIRT) += paravirt_patch_64.o endif +obj-$(CONFIG_PARAVIRT) += paravirt.o # Workaround to delete .lds files with make clean # The problem is that we do not enter Makefile_32 with make clean.
arch/x86/kernel/Makefile_32 +0 -1
@@ -43,7 +43,6 @@ obj-$(CONFIG_K8_NB) += k8.o obj-$(CONFIG_MGEODE_LX) += geode_32.o mfgpt_32.o obj-$(CONFIG_VMI) += vmi_32.o vmiclock_32.o -obj-$(CONFIG_PARAVIRT) += paravirt_32.o obj-y += pcspeaker.o obj-$(CONFIG_SCx200) += scx200_32.o
arch/x86/kernel/asm-offsets.c +8 -0
@@ -1,3 +1,11 @@ +#define DEFINE (sym, val) \ + asm volatile("\n->" #sym " %0 " #val : : "i" (val)) + +#define BLANK () asm volatile("\n->" : : ) + +#define OFFSET (sym, str, mem) \ + DEFINE(sym, offsetof(struct str, mem)); +#ifdef CONFIG_X86_32 # include "asm-offsets_32.c" #else
arch/x86/kernel/asm-offsets_32.c +0 -8
@@ -25,14 +25,6 @@#include "../../../drivers/lguest/lg.h" #endif -#define DEFINE (sym, val) \ - asm volatile("\n->" #sym " %0 " #val : : "i" (val)) - -#define BLANK () asm volatile("\n->" : : ) - -#define OFFSET (sym, str, mem) \ - DEFINE(sym, offsetof(struct str, mem)); -/* workaround for a warning with -Wmissing-prototypes */ void foo (void );
arch/x86/kernel/asm-offsets_64.c +14 -8
@@ -16,14 +16,7 @@#include <asm/thread_info.h> #include <asm/ia32.h> #include <asm/bootparam.h> - -#define DEFINE (sym, val) \ - asm volatile("\n->" #sym " %0 " #val : : "i" (val)) - -#define BLANK () asm volatile("\n->" : : ) - -#define OFFSET (sym, str, mem) \ - DEFINE(sym, offsetof(struct str, mem)) +#include <asm/paravirt.h> #define __NO_STUBS 1#undef __SYSCALL
@@ -76,6 +69,19 @@ int main(void) offsetof (struct rt_sigframe32, uc.uc_mcontext));BLANK ();#endif +#ifdef CONFIG_PARAVIRT +OFFSET (PARAVIRT_enabled, pv_info, paravirt_enabled); +OFFSET (PARAVIRT_PATCH_pv_cpu_ops, paravirt_patch_template, pv_cpu_ops); +OFFSET (PARAVIRT_PATCH_pv_irq_ops, paravirt_patch_template, pv_irq_ops); +OFFSET (PARAVIRT_PATCH_pv_mmu_ops, paravirt_patch_template, pv_mmu_ops); +OFFSET (PV_IRQ_irq_disable, pv_irq_ops, irq_disable); +OFFSET (PV_IRQ_irq_enable, pv_irq_ops, irq_enable); +OFFSET (PV_CPU_iret, pv_cpu_ops, iret); +OFFSET (PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret); +OFFSET (PV_MMU_read_cr2, pv_mmu_ops, read_cr2); +OFFSET (PV_CPU_swapgs, pv_cpu_ops, swapgs); +BLANK (); +#endif DEFINE (pbe_address, offsetof(struct pbe , address));DEFINE (pbe_orig_address, offsetof(struct pbe , orig_address));DEFINE (pbe_next, offsetof(struct pbe , next));
arch/x86/kernel/paravirt.c new file +445 -0
445 lines not on this page · raw message
arch/x86/kernel/paravirt_32.c deleted +0 -472
472 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_32.c new file +52 -0
52 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_64.c new file +56 -0
56 lines not on this page · raw message
arch/x86/kernel/vmlinux_64.lds.S +6 -0
@@ -185,6 +185,12 @@ SECTIONS .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) { *(.altinstr_replacement) } + . = ALIGN(8); + .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) { + __parainstructions = .; + *(.parainstructions) + __parainstructions_end = .; + } /* .exit.text is discard at runtime, not link time, to deal with references from .altinstructions and .eh_frame */ .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) { *(.exit.text) }
include/asm-x86/paravirt.h +376 -82
806 lines not on this page · raw message
Signature · 8 lines
1.4.4.2 ------------------------------------------------------------------------- This SF.net email is sponsored by: Splunk Inc. Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
PATCH 16/16 make vsmp a paravirt client
This patch makes vsmp a paravirt client. It now uses the whole infrastructure provided by pvops. When we detect we're running a vsmp box, we change the irq-related paravirt operations (and so, it have to happen quite early), and the patching function
--- arch/x86/Kconfig.x86_64 | 3 +- arch/x86/kernel/setup_64.c | 3 ++ arch/x86/kernel/vsmp_64.c | 72 +++++++++++++++++++++++++++++++++++++++---- include/asm-x86/setup.h | 3 +- 4 files changed, 71 insertions(+), 10 deletions(-)
arch/x86/Kconfig.x86_64 +1 -2
@@ -148,15 +148,14 @@ config X86_PC bool "PC-compatible" help Choose this option if your computer is a standard PC or compatible. - config X86_VSMP bool "Support for ScaleMP vSMP" depends on PCI + select PARAVIRT help Support for ScaleMP vSMP systems. Say 'Y' here if this kernel is supposed to run on these EM64T-based machines. Only choose this option if you have one of these machines. - endchoice choice
arch/x86/kernel/setup_64.c +3 -0
@@ -335,6 +335,9 @@ void __init setup_arch(char **cmdline_p)init_memory_mapping (0 , (end_pfn_map <<PAGE_SHIFT )); +#ifdef CONFIG_VSMP +vsmp_init (); +#endif dmi_scan_machine ();#ifdef CONFIG_SMP
arch/x86/kernel/vsmp_64.c +65 -7
99 lines not on this page · raw message
include/asm-x86/setup.h +2 -1
@@ -58,7 +58,8 @@ void __init add_memory_region(unsigned long long start,extern unsigned long init_pg_tables_end; - +/* For EM64T-based VSMP machines */ +int vsmp_init (void );#endif /* __i386__ */ #endif /* _SETUP */
Signature · 8 lines
1.4.4.2 ------------------------------------------------------------------------- This SF.net email is sponsored by: Splunk Inc. Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [PATCH 16/16] make vsmp a paravirt client
Quoted 64 lines · Glauber de Oliveira Costa
This patch makes vsmp a paravirt client. It now uses the whole infrastructure provided by pvops. When we detect we're running a vsmp box, we change the irq-related paravirt operations (and so, it have to happen quite early), and the patching function
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/Kconfig.x86_64 | 3 +- arch/x86/kernel/setup_64.c | 3 ++ arch/x86/kernel/vsmp_64.c | 72 +++++++++++++++++++++++++++++++++++++++---- include/asm-x86/setup.h | 3 +- 4 files changed, 71 insertions(+), 10 deletions(-)arch/x86/Kconfig.x86_64 +1
-2@@ -148,15 +148,14 @@ config X86_PC bool "PC-compatible" help Choose this option if your computer is a standard PC or compatible. - config X86_VSMP bool "Support for ScaleMP vSMP" depends on PCI + select PARAVIRT help Support for ScaleMP vSMP systems. Say 'Y' here if this kernel is supposed to run on these EM64T-based machines. Only choose this option if you have one of these machines. - endchoice choicearch/x86/kernel/setup_64.c +3
-0@@ -335,6 +335,9 @@ void __init setup_arch(char **cmdline_p)init_memory_mapping (0 , (end_pfn_map <<PAGE_SHIFT )); +#ifdef CONFIG_VSMP +vsmp_init (); +#endif dmi_scan_machine ();#ifdef CONFIG_SMP arch/x86/kernel/vsmp_64.c +1
-1@@ -8,18 +8,70 @@* * Ravikiran Thirumalai <kiran-Yzr3oMT/[email protected]>, * Shai Fultheim <shai-Yzr3oMT/[email protected]> +* Paravirt ops integration: Glauber de Oliveira Costa <[email protected]> */ -#include <linux/init.h> #include <linux/pci_ids.h> #include <linux/pci_regs.h>
arch/x86/kernel/vsmp_64.c line 25
Quoted 10 lines
arch/x86/kernel/vsmp_64.c +7
-0#include <asm/pci-direct.h> #include <asm/io.h> +#include <asm/paravirt.h> + +/* +* Interrupt control for the VSMP architecture: +*/ + +static inline unsigned long vsmp_save_fl (void )
No point being inline.
arch/x86/kernel/vsmp_64.c line 32
Quoted 6 lines
arch/x86/kernel/vsmp_64.c +5
-0+{ +unsigned long flags =native_save_fl (); + +if (flags &X86_EFLAGS_IF ) +return X86_EFLAGS_IF ;
Is this right, or should the if be testing _AC? Otherwise, why not just "return flags & X86_EFLAGS_IF"?
arch/x86/kernel/vsmp_64.c line 75
Quoted 11 lines
arch/x86/kernel/vsmp_64.c +8
-1+return 0 ; +} -static int __init vsmp_init (void ) +static inline void vsmp_restore_fl (unsigned long flags) +{ +if (flags &X86_EFLAGS_IF ) + flags &= ~X86_EFLAGS_AC ; +if (!(flags &X86_EFLAGS_IF )) + flags &=X86_EFLAGS_AC ;
Just use "else"?
arch/x86/kernel/vsmp_64.c line 75
Quoted 9 lines
arch/x86/kernel/vsmp_64.c +8
-0+native_restore_fl (flags); +} + +static inline void vsmp_irq_disable (void ) +{ +unsigned long flags =native_save_fl (); + +vsmp_restore_fl ((flags & ~X86_EFLAGS_IF ));
((double paren?))
Quoted 66 lines
arch/x86/kernel/vsmp_64.c +36
-5+} + +static inline void vsmp_irq_enable (void ) +{ +unsigned long flags =native_save_fl (); + +vsmp_restore_fl ((flags |X86_EFLAGS_IF )); +} + +static unsigned __init vsmp_patch (u8 type,u16 clobbers,void *ibuf, +unsigned long addr,unsigned len) +{ +switch (type) { +case PARAVIRT_PATCH (pv_irq_ops.irq_enable): +case PARAVIRT_PATCH (pv_irq_ops.irq_disable): +case PARAVIRT_PATCH (pv_irq_ops.save_fl): +case PARAVIRT_PATCH (pv_irq_ops.restore_fl): +return paravirt_patch_default (type, clobbers, ibuf, addr, len); +default : +return native_patch (type, clobbers, ibuf, addr, len); + } + +} + +int __init vsmp_init (void ) {void *address; -unsigned int cap, ctl; +unsigned int cap, ctl, cfg;if (!early_pci_allowed ())return 0 ;@@ -29,8 +81,16 @@ static int __init vsmp_init(void) (read_pci_config_16(0, 0x1f, 0, PCI_DEVICE_ID) != PCI_DEVICE_ID_SCALEMP_VSMP_CTL))return 0 ; +/* If we are, use the distinguished irq functions */ + pv_irq_ops.irq_disable = vsmp_irq_disable; + pv_irq_ops.irq_enable = vsmp_irq_enable; + pv_irq_ops.save_fl = vsmp_save_fl; + pv_irq_ops.restore_fl = vsmp_restore_fl; + pv_init_ops.patch = vsmp_patch; +/* set vSMP magic bits to indicate vSMP capable kernel */ - address =ioremap (read_pci_config (0 ,0x1f ,0 ,PCI_BASE_ADDRESS_0 ),8 ); + cfg =read_pci_config (0 ,0x1f ,0 ,PCI_BASE_ADDRESS_0 ); + address =early_ioremap (cfg,8 ); cap =readl (address); ctl =readl (address +4 );printk ("vSMP CTL: capabilities:0x%08x control:0x%08x\n" , cap, ctl);@@ -42,8 +102,6 @@ static int __init vsmp_init(void)printk ("vSMP CTL: control set to:0x%08x\n" , ctl); } -iounmap (address); +early_iounmap (address,8 );return 0 ; } - -core_initcall (vsmp_init);include/asm-x86/setup.h +0
-0
include/asm-x86/setup.h void __init add_memory_region(unsigned long long start, · line 65
Quoted 9 lines
include/asm-x86/setup.h +2
-1@@ -58,7 +58,8 @@ void __init add_memory_region(unsigned long long start,extern unsigned long init_pg_tables_end; - +/* For EM64T-based VSMP machines */ +int vsmp_init (void );#endif /* __i386__ */ #endif /* _SETUP */
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [PATCH 16/16] make vsmp a paravirt client
In reply to Glauber de Oliveira CostaQuoted 64 lines · Glauber de Oliveira Costa
This patch makes vsmp a paravirt client. It now uses the whole infrastructure provided by pvops. When we detect we're running a vsmp box, we change the irq-related paravirt operations (and so, it have to happen quite early), and the patching function
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/Kconfig.x86_64 | 3 +- arch/x86/kernel/setup_64.c | 3 ++ arch/x86/kernel/vsmp_64.c | 72 +++++++++++++++++++++++++++++++++++++++---- include/asm-x86/setup.h | 3 +- 4 files changed, 71 insertions(+), 10 deletions(-)arch/x86/Kconfig.x86_64 +1
-2@@ -148,15 +148,14 @@ config X86_PC bool "PC-compatible" help Choose this option if your computer is a standard PC or compatible. - config X86_VSMP bool "Support for ScaleMP vSMP" depends on PCI + select PARAVIRT help Support for ScaleMP vSMP systems. Say 'Y' here if this kernel is supposed to run on these EM64T-based machines. Only choose this option if you have one of these machines. - endchoice choicearch/x86/kernel/setup_64.c +3
-0@@ -335,6 +335,9 @@ void __init setup_arch(char **cmdline_p)init_memory_mapping (0 , (end_pfn_map <<PAGE_SHIFT )); +#ifdef CONFIG_VSMP +vsmp_init (); +#endif dmi_scan_machine ();#ifdef CONFIG_SMP arch/x86/kernel/vsmp_64.c +1
-1@@ -8,18 +8,70 @@* * Ravikiran Thirumalai <[email protected]>, * Shai Fultheim <[email protected]> +* Paravirt ops integration: Glauber de Oliveira Costa <[email protected]> */ -#include <linux/init.h> #include <linux/pci_ids.h> #include <linux/pci_regs.h>
arch/x86/kernel/vsmp_64.c line 25
Quoted 10 lines
arch/x86/kernel/vsmp_64.c +7
-0#include <asm/pci-direct.h> #include <asm/io.h> +#include <asm/paravirt.h> + +/* +* Interrupt control for the VSMP architecture: +*/ + +static inline unsigned long vsmp_save_fl (void )
No point being inline.
arch/x86/kernel/vsmp_64.c line 32
Quoted 6 lines
arch/x86/kernel/vsmp_64.c +5
-0+{ +unsigned long flags =native_save_fl (); + +if (flags &X86_EFLAGS_IF ) +return X86_EFLAGS_IF ;
Is this right, or should the if be testing _AC? Otherwise, why not just "return flags & X86_EFLAGS_IF"?
arch/x86/kernel/vsmp_64.c line 75
Quoted 11 lines
arch/x86/kernel/vsmp_64.c +8
-1+return 0 ; +} -static int __init vsmp_init (void ) +static inline void vsmp_restore_fl (unsigned long flags) +{ +if (flags &X86_EFLAGS_IF ) + flags &= ~X86_EFLAGS_AC ; +if (!(flags &X86_EFLAGS_IF )) + flags &=X86_EFLAGS_AC ;
Just use "else"?
arch/x86/kernel/vsmp_64.c line 75
Quoted 9 lines
arch/x86/kernel/vsmp_64.c +8
-0+native_restore_fl (flags); +} + +static inline void vsmp_irq_disable (void ) +{ +unsigned long flags =native_save_fl (); + +vsmp_restore_fl ((flags & ~X86_EFLAGS_IF ));
((double paren?))
Quoted 75 lines
arch/x86/kernel/vsmp_64.c +36
-5+} + +static inline void vsmp_irq_enable (void ) +{ +unsigned long flags =native_save_fl (); + +vsmp_restore_fl ((flags |X86_EFLAGS_IF )); +} + +static unsigned __init vsmp_patch (u8 type,u16 clobbers,void *ibuf, +unsigned long addr,unsigned len) +{ +switch (type) { +case PARAVIRT_PATCH (pv_irq_ops.irq_enable): +case PARAVIRT_PATCH (pv_irq_ops.irq_disable): +case PARAVIRT_PATCH (pv_irq_ops.save_fl): +case PARAVIRT_PATCH (pv_irq_ops.restore_fl): +return paravirt_patch_default (type, clobbers, ibuf, addr, len); +default : +return native_patch (type, clobbers, ibuf, addr, len); + } + +} + +int __init vsmp_init (void ) {void *address; -unsigned int cap, ctl; +unsigned int cap, ctl, cfg;if (!early_pci_allowed ())return 0 ;@@ -29,8 +81,16 @@ static int __init vsmp_init(void) (read_pci_config_16(0, 0x1f, 0, PCI_DEVICE_ID) != PCI_DEVICE_ID_SCALEMP_VSMP_CTL))return 0 ; +/* If we are, use the distinguished irq functions */ + pv_irq_ops.irq_disable = vsmp_irq_disable; + pv_irq_ops.irq_enable = vsmp_irq_enable; + pv_irq_ops.save_fl = vsmp_save_fl; + pv_irq_ops.restore_fl = vsmp_restore_fl; + pv_init_ops.patch = vsmp_patch; +/* set vSMP magic bits to indicate vSMP capable kernel */ - address =ioremap (read_pci_config (0 ,0x1f ,0 ,PCI_BASE_ADDRESS_0 ),8 ); + cfg =read_pci_config (0 ,0x1f ,0 ,PCI_BASE_ADDRESS_0 ); + address =early_ioremap (cfg,8 ); cap =readl (address); ctl =readl (address +4 );printk ("vSMP CTL: capabilities:0x%08x control:0x%08x\n" , cap, ctl);@@ -42,8 +102,6 @@ static int __init vsmp_init(void)printk ("vSMP CTL: control set to:0x%08x\n" , ctl); } -iounmap (address); +early_iounmap (address,8 );return 0 ; } - -core_initcall (vsmp_init);include/asm-x86/setup.h +2
-1@@ -58,7 +58,8 @@ void __init add_memory_region(unsigned long long start,extern unsigned long init_pg_tables_end; - +/* For EM64T-based VSMP machines */ +int vsmp_init (void );#endif /* __i386__ */ #endif /* _SETUP */
PATCH 16/16 make vsmp a paravirt client
In reply to Glauber de Oliveira CostaThis patch makes vsmp a paravirt client. It now uses the whole infrastructure provided by pvops. When we detect we're running a vsmp box, we change the irq-related paravirt operations (and so, it have to happen quite early), and the patching function
--- arch/x86/Kconfig.x86_64 | 3 +- arch/x86/kernel/setup_64.c | 3 ++ arch/x86/kernel/vsmp_64.c | 72 +++++++++++++++++++++++++++++++++++++++---- include/asm-x86/setup.h | 3 +- 4 files changed, 71 insertions(+), 10 deletions(-)
arch/x86/Kconfig.x86_64 +1 -2
@@ -148,15 +148,14 @@ config X86_PC bool "PC-compatible" help Choose this option if your computer is a standard PC or compatible. - config X86_VSMP bool "Support for ScaleMP vSMP" depends on PCI + select PARAVIRT help Support for ScaleMP vSMP systems. Say 'Y' here if this kernel is supposed to run on these EM64T-based machines. Only choose this option if you have one of these machines. - endchoice choice
arch/x86/kernel/setup_64.c +3 -0
@@ -335,6 +335,9 @@ void __init setup_arch(char **cmdline_p)init_memory_mapping (0 , (end_pfn_map <<PAGE_SHIFT )); +#ifdef CONFIG_VSMP +vsmp_init (); +#endif dmi_scan_machine ();#ifdef CONFIG_SMP
arch/x86/kernel/vsmp_64.c +65 -7
99 lines not on this page · raw message
include/asm-x86/setup.h +2 -1
@@ -58,7 +58,8 @@ void __init add_memory_region(unsigned long long start,extern unsigned long init_pg_tables_end; - +/* For EM64T-based VSMP machines */ +int vsmp_init (void );#endif /* __i386__ */ #endif /* _SETUP */
Signature · 1 line
1.4.4.2
PATCH 15/16 consolidation of paravirt for 32 and 64 bits
In reply to Glauber de Oliveira CostaThis patch unifies the paravirt ops structures for usage in x86_64 and i386 architectures. Some new field had to be created to accomodate the differences between the architectures.
--- arch/x86/Kconfig.x86_64 | 11 + arch/x86/kernel/Makefile | 3 + arch/x86/kernel/Makefile_32 | 1 - arch/x86/kernel/asm-offsets.c | 8 + arch/x86/kernel/asm-offsets_32.c | 8 - arch/x86/kernel/asm-offsets_64.c | 22 +- arch/x86/kernel/paravirt.c | 445 +++++++++++++++++++++++++++++++++ arch/x86/kernel/paravirt_32.c | 472 ----------------------------------- arch/x86/kernel/paravirt_patch_32.c | 52 ++++ arch/x86/kernel/paravirt_patch_64.c | 56 ++++ arch/x86/kernel/vmlinux_64.lds.S | 6 + include/asm-x86/paravirt.h | 458 ++++++++++++++++++++++++++++------ 12 files changed, 971 insertions(+), 571 deletions(-)
arch/x86/Kconfig.x86_64 +11 -0
@@ -372,6 +372,17 @@ config NODES_SHIFT # Dummy CONFIG option to select ACPI_NUMA from drivers/acpi/Kconfig. +config PARAVIRT + bool + depends on EXPERIMENTAL + help + Paravirtualization is a way of running multiple instances of + Linux on the same machine, under a hypervisor. This option + changes the kernel so it can modify itself when it is run + under a hypervisor, improving performance significantly. + However, when run without a hypervisor the kernel is + theoretically slower. If in doubt, say N. + config X86_64_ACPI_NUMA bool "ACPI NUMA detection" depends on NUMA
arch/x86/kernel/Makefile +3 -0
@@ -1,8 +1,11 @@ ifeq ($(CONFIG_X86_32),y) include ${srctree}/arch/x86/kernel/Makefile_32 +obj-$(CONFIG_PARAVIRT) += paravirt_patch_32.o else include ${srctree}/arch/x86/kernel/Makefile_64 +obj-$(CONFIG_PARAVIRT) += paravirt_patch_64.o endif +obj-$(CONFIG_PARAVIRT) += paravirt.o # Workaround to delete .lds files with make clean # The problem is that we do not enter Makefile_32 with make clean.
arch/x86/kernel/Makefile_32 +0 -1
@@ -43,7 +43,6 @@ obj-$(CONFIG_K8_NB) += k8.o obj-$(CONFIG_MGEODE_LX) += geode_32.o mfgpt_32.o obj-$(CONFIG_VMI) += vmi_32.o vmiclock_32.o -obj-$(CONFIG_PARAVIRT) += paravirt_32.o obj-y += pcspeaker.o obj-$(CONFIG_SCx200) += scx200_32.o
arch/x86/kernel/asm-offsets.c +8 -0
@@ -1,3 +1,11 @@ +#define DEFINE (sym, val) \ + asm volatile("\n->" #sym " %0 " #val : : "i" (val)) + +#define BLANK () asm volatile("\n->" : : ) + +#define OFFSET (sym, str, mem) \ + DEFINE(sym, offsetof(struct str, mem)); +#ifdef CONFIG_X86_32 # include "asm-offsets_32.c" #else
arch/x86/kernel/asm-offsets_32.c +0 -8
@@ -25,14 +25,6 @@#include "../../../drivers/lguest/lg.h" #endif -#define DEFINE (sym, val) \ - asm volatile("\n->" #sym " %0 " #val : : "i" (val)) - -#define BLANK () asm volatile("\n->" : : ) - -#define OFFSET (sym, str, mem) \ - DEFINE(sym, offsetof(struct str, mem)); -/* workaround for a warning with -Wmissing-prototypes */ void foo (void );
arch/x86/kernel/asm-offsets_64.c +14 -8
@@ -16,14 +16,7 @@#include <asm/thread_info.h> #include <asm/ia32.h> #include <asm/bootparam.h> - -#define DEFINE (sym, val) \ - asm volatile("\n->" #sym " %0 " #val : : "i" (val)) - -#define BLANK () asm volatile("\n->" : : ) - -#define OFFSET (sym, str, mem) \ - DEFINE(sym, offsetof(struct str, mem)) +#include <asm/paravirt.h> #define __NO_STUBS 1#undef __SYSCALL
@@ -76,6 +69,19 @@ int main(void) offsetof (struct rt_sigframe32, uc.uc_mcontext));BLANK ();#endif +#ifdef CONFIG_PARAVIRT +OFFSET (PARAVIRT_enabled, pv_info, paravirt_enabled); +OFFSET (PARAVIRT_PATCH_pv_cpu_ops, paravirt_patch_template, pv_cpu_ops); +OFFSET (PARAVIRT_PATCH_pv_irq_ops, paravirt_patch_template, pv_irq_ops); +OFFSET (PARAVIRT_PATCH_pv_mmu_ops, paravirt_patch_template, pv_mmu_ops); +OFFSET (PV_IRQ_irq_disable, pv_irq_ops, irq_disable); +OFFSET (PV_IRQ_irq_enable, pv_irq_ops, irq_enable); +OFFSET (PV_CPU_iret, pv_cpu_ops, iret); +OFFSET (PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret); +OFFSET (PV_MMU_read_cr2, pv_mmu_ops, read_cr2); +OFFSET (PV_CPU_swapgs, pv_cpu_ops, swapgs); +BLANK (); +#endif DEFINE (pbe_address, offsetof(struct pbe , address));DEFINE (pbe_orig_address, offsetof(struct pbe , orig_address));DEFINE (pbe_next, offsetof(struct pbe , next));
arch/x86/kernel/paravirt.c new file +445 -0
445 lines not on this page · raw message
arch/x86/kernel/paravirt_32.c deleted +0 -472
472 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_32.c new file +52 -0
52 lines not on this page · raw message
arch/x86/kernel/paravirt_patch_64.c new file +56 -0
56 lines not on this page · raw message
arch/x86/kernel/vmlinux_64.lds.S +6 -0
@@ -185,6 +185,12 @@ SECTIONS .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) { *(.altinstr_replacement) } + . = ALIGN(8); + .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) { + __parainstructions = .; + *(.parainstructions) + __parainstructions_end = .; + } /* .exit.text is discard at runtime, not link time, to deal with references from .altinstructions and .eh_frame */ .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) { *(.exit.text) }
include/asm-x86/paravirt.h +376 -82
806 lines not on this page · raw message
Signature · 1 line
1.4.4.2
PATCH 14/16 prepare x86_64 architecture initialization for paravirt
In reply to Glauber de Oliveira CostaThis patch prepares the x86_64 architecture initialization for paravirt. It requires a memory initialization step, which is done by implementing 64-bit version for machine_specific_memory_setup, and putting an ARCH_SETUP hook, for guest-dependent initialization. This last step is done akin to i386
--- arch/x86/kernel/e820_64.c | 9 +++++++-- arch/x86/kernel/setup_64.c | 28 +++++++++++++++++++++++++++- include/asm-x86/setup.h | 11 ++++++++--- 3 files changed, 42 insertions(+), 6 deletions(-)
arch/x86/kernel/e820_64.c +7 -2
@@ -593,8 +593,10 @@ void early_panic(char *msg)panic (msg); } -void __init setup_memory_region (void ) +/* We're not void only for x86 32-bit compat */ +char *__init machine_specific_memory_setup (void ) { +char *who ="BIOS-e820" ;/* * Try to copy the BIOS-supplied E820-map. *
@@ -605,7 +607,10 @@ void __init setup_memory_region(void)if (copy_e820_map (boot_params.e820_map, boot_params.e820_entries) <0 )early_panic ("Cannot find a valid memory map" );printk (KERN_INFO "BIOS-provided physical RAM map:\n" ); -e820_print_map ("BIOS-e820" ); +e820_print_map (who); + +/* In case someone cares... */ +return who; }static int __init parse_memopt (char *p)
arch/x86/kernel/setup_64.c +27 -1
@@ -39,6 +39,7 @@#include <linux/dmi.h> #include <linux/dma-mapping.h> #include <linux/ctype.h> +#include <linux/uaccess.h> #include <asm/mtrr.h> #include <asm/uaccess.h>
@@ -60,6 +61,12 @@#include <asm/dmi.h> #include <asm/cacheflush.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define ARCH_SETUP +#endif +/* * Machine setup.. */
@@ -241,6 +248,16 @@ static void discover_ebda(void)* 4K EBDA area at 0x40E */ ebda_addr = *(unsigned short *)__va (EBDA_ADDR_POINTER ); +/* +* There can be some situations, like paravirtualized guests, +* in which there is no available ebda information. In such +* case, just skip it +*/ +if (!ebda_addr) { + ebda_size =0 ; +return ; + } + ebda_addr <<=4 ; ebda_size = *(unsigned short *)__va (ebda_addr);
@@ -254,6 +271,12 @@ static void discover_ebda(void) ebda_size =64 *1024 ; } +/* Overridden in paravirt.c if CONFIG_PARAVIRT */ +void __attribute__((weak))memory_setup (void ) +{ +machine_specific_memory_setup (); +} +void __init setup_arch (char **cmdline_p) {printk (KERN_INFO "Command line: %s\n" , boot_command_line);
@@ -269,7 +292,10 @@ void __init setup_arch(char **cmdline_p) rd_prompt = ((boot_params.hdr.ram_size &RAMDISK_PROMPT_FLAG ) !=0 ); rd_doload = ((boot_params.hdr.ram_size &RAMDISK_LOAD_FLAG ) !=0 );#endif -setup_memory_region (); + +ARCH_SETUP + +memory_setup ();copy_edd ();if (!boot_params.hdr.root_flags)
include/asm-x86/setup.h +8 -3
@@ -3,6 +3,13 @@#define COMMAND_LINE_SIZE 2048 +#ifndef __ASSEMBLY__ +char *machine_specific_memory_setup (void ); +#ifndef CONFIG_PARAVIRT +#define paravirt_post_allocator_init () do {} while (0) +#endif +#endif /* __ASSEMBLY__ */ +#ifdef __KERNEL__#ifdef __i386__
@@ -51,9 +58,7 @@ void __init add_memory_region(unsigned long long start,extern unsigned long init_pg_tables_end; -#ifndef CONFIG_PARAVIRT -#define paravirt_post_allocator_init () do {} while (0) -#endif +#endif /* __i386__ */ #endif /* _SETUP */
Signature · 1 line
1.4.4.2
PATCH 13/16 native versions for page table entries values
In reply to Glauber de Oliveira CostaThis patch turns the page operations (set and make a page table) into native_ versions. The operations itself will be later overriden by paravirt.
It uses unsigned long long for consistency with 32-bit. So we have to fix fault_64.c to get rid of warnings.
--- arch/x86/mm/fault_64.c | 8 +++--- include/asm-x86/page_64.h | 56 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 9 deletions(-)
arch/x86/mm/fault_64.c +4 -4
@@ -158,22 +158,22 @@ void dump_pagetable(unsigned long address) pgd =__va ((unsigned long )pgd &PHYSICAL_PAGE_MASK ); pgd +=pgd_index (address);if (bad_address (pgd)) goto bad; -printk ("PGD %lx " ,pgd_val (*pgd)); +printk ("PGD %llx " ,pgd_val (*pgd));if (!pgd_present (*pgd)) goto ret; pud =pud_offset (pgd, address);if (bad_address (pud)) goto bad; -printk ("PUD %lx " ,pud_val (*pud)); +printk ("PUD %llx " ,pud_val (*pud));if (!pud_present (*pud)) goto ret; pmd =pmd_offset (pud, address);if (bad_address (pmd)) goto bad; -printk ("PMD %lx " ,pmd_val (*pmd)); +printk ("PMD %llx " ,pmd_val (*pmd));if (!pmd_present (*pmd) ||pmd_large (*pmd)) goto ret; pte =pte_offset_kernel (pmd, address);if (bad_address (pte)) goto bad; -printk ("PTE %lx" ,pte_val (*pte)); +printk ("PTE %llx" ,pte_val (*pte)); ret:printk ("\n" );return ;
include/asm-x86/page_64.h +51 -5
67 lines not on this page · raw message
Signature · 1 line
1.4.4.2
PATCH 12/16 tweak io_64.h for paravirt.
In reply to Glauber de Oliveira CostaWe need something here because we can't call in and out instructions directly. However, we have to be careful, because no indirections are allowed in misc_64.c , and paravirt_ops is a kind of one. So just call it directly there
--- arch/x86/boot/compressed/misc_64.c | 6 +++++ include/asm-x86/io_64.h | 37 +++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-)
arch/x86/boot/compressed/misc_64.c +6 -0
@@ -9,6 +9,12 @@* High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996 */ +/* +* we have to be careful, because no indirections are allowed here, and +* paravirt_ops is a kind of one. As it will only run in baremetal anyway, +* we just keep it from happening +*/ +#undef CONFIG_PARAVIRT#define _LINUX_STRING_H_ 1#define __LINUX_BITMAP_H 1
include/asm-x86/io_64.h +30 -7
@@ -35,12 +35,24 @@* - Arnaldo Carvalho de Melo <[email protected]> */ -#define __SLOW_DOWN_IO "\noutb %%al,$0x80" +static inline void native_io_delay (void ) +{ + asmvolatile ("outb %%al,$0x80" : : :"memory" ); +} -#ifdef REALLY_SLOW_IO -#define __FULL_SLOW_DOWN_IO __SLOW_DOWN_IO __SLOW_DOWN_IO __SLOW_DOWN_IO __SLOW_DOWN_IO +#if defined(CONFIG_PARAVIRT ) +#include <asm/paravirt.h> #else -#define __FULL_SLOW_DOWN_IO __SLOW_DOWN_IO + +static inline void slow_down_io (void ) +{ +native_io_delay (); +#ifdef REALLY_SLOW_IO +native_io_delay (); +native_io_delay (); +native_io_delay (); +#endif +}#endif /*
@@ -52,9 +64,15 @@ static inline void out##s(unsigned x value, unsigned short port) {#define __OUT2 (s,s1,s2) \ __asm__ __volatile__ ("out" #s " %" s1 "0,%" s2 "1" +#ifndef REALLY_SLOW_IO +#define REALLY_SLOW_IO +#define UNSET_REALLY_SLOW_IO +#endif +#define __OUT (s,s1,x) \ __OUT1(s,x) __OUT2(s,s1,"w") : : "a" (value), "Nd" (port)); } \ -__OUT1(s##_p,x) __OUT2(s,s1,"w") __FULL_SLOW_DOWN_IO : : "a" (value), "Nd" (port));} \ +__OUT1(s##_p, x) __OUT2(s, s1, "w") : : "a" (value), "Nd" (port)); \ + slow_down_io(); }#define __IN1 (s) \ static inline RETURN_TYPE in##s(unsigned short port) { RETURN_TYPE _v;
@@ -63,8 +81,13 @@ static inline RETURN_TYPE in##s(unsigned short port) { RETURN_TYPE _v; __asm__ __volatile__ ("in" #s " %" s2 "1,%" s1 "0" #define __IN(s,s1,i...) \ -__IN1(s) __IN2(s,s1,"w") : "=a" (_v) : "Nd" (port) ,##i ); return _v; } \ -__IN1(s##_p) __IN2(s,s1,"w") __FULL_SLOW_DOWN_IO : "=a" (_v) : "Nd" (port) ,##i ); return _v; } \ +__IN1(s) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); return _v; } \ +__IN1(s##_p) __IN2(s, s1, "w") : "=a" (_v) : "Nd" (port), ##i); \ + slow_down_io(); return _v; } + +#ifdef UNSET_REALLY_SLOW_IO +#undef REALLY_SLOW_IO +#endif #define __INS (s) \ static inline void ins##s(unsigned short port, void * addr, unsigned long count) \
Signature · 1 line
1.4.4.2
Re: [PATCH 11/16] turn priviled operation into a macro in head_64.S
In reply to Glauber de Oliveira CostaQuoted 5 lines · Glauber de Oliveira Costa
under paravirt, read cr2 cannot be issued directly anymore. So wrap it in a macro, defined to the operation itself in case paravirt is off, but to something else if we have paravirt in the game
Is this actually needed? It's only used in the early fault handler in head_64.S. Will we be taking that path in the paravirt case? If so, should we disable the fault handler altogether, since the hypervisor can probably provide better diagnositcs.
J
Quoted 27 lines
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/kernel/head_64.S | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-)arch/x86/kernel/head_64.S +7
-0@@ -19,6 +19,13 @@ #include <asm/msr.h> #include <asm/cache.h> +#ifdef CONFIG_PARAVIRT +#include <asm/asm-offsets.h> +#include <asm/paravirt.h> +#else +#define GET_CR2_INTO_RCX movq %cr2, %rcx +#endif + /* we are not able to switch in one step to the final KERNEL ADRESS SPACE * because we need identity-mapped pages. *
arch/x86/kernel/head_64.S ENTRY(early_idt_handler) · line 280
Quoted 8 lines
arch/x86/kernel/head_64.S +1
-1@@ -267,7 +274,7 @@ ENTRY(early_idt_handler) xorl %eax,%eax movq 8(%rsp),%rsi # get rip movq (%rsp),%rdx - movq %cr2,%rcx + GET_CR2_INTO_RCX leaq early_idt_msg(%rip),%rdi call early_printk cmpl $2,early_recursion_flag(%rip)
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [PATCH 11/16] turn priviled operation into a macro in head_64.S
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Jeremy Fitzhardinge escreveu:
Quoted 7 lines
Quoted 5 lines · Glauber de Oliveira Costa
under paravirt, read cr2 cannot be issued directly anymore. So wrap it in a macro, defined to the operation itself in case paravirt is off, but to something else if we have paravirt in the game
Quoted 5 lines
Is this actually needed? It's only used in the early fault handler in
head_64.S. Will we be taking that path in the paravirt case? If so, should we disable the fault handler altogether, since the hypervisor can probably provide better diagnositcs.
Well, as you told me earlier, xen won't use it. Neither does lguest. None of us goes through the normal boot process anyway. But maybe some other technology uses it?
Zach, how does it work for vmware?
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.7 (GNU/Linux)
Comment: Using GnuPG with Remi - http://enigmail.mozdev.org
iD8DBQFHKdmVjYI8LaFUWXMRAgxkAKCQB+VtpGSlm+zuyRRWmi3h+k9NqQCgyfmD cpPyQcfxo9hcSI0WaDFmpWg=
=Oa4v
-----END PGP SIGNATURE-----
Re: [PATCH 11/16] turn priviled operation into a macro in head_64.S
In reply to Jeremy Fitzhardinge-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Jeremy Fitzhardinge escreveu:
Quoted 7 lines
Quoted 5 lines · Glauber de Oliveira Costa
under paravirt, read cr2 cannot be issued directly anymore. So wrap it in a macro, defined to the operation itself in case paravirt is off, but to something else if we have paravirt in the game
Quoted 5 lines
Is this actually needed? It's only used in the early fault handler in
head_64.S. Will we be taking that path in the paravirt case? If so, should we disable the fault handler altogether, since the hypervisor can probably provide better diagnositcs.
Well, as you told me earlier, xen won't use it. Neither does lguest. None of us goes through the normal boot process anyway. But maybe some other technology uses it?
Zach, how does it work for vmware?
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.7 (GNU/Linux)
Comment: Using GnuPG with Remi - http://enigmail.mozdev.org
iD8DBQFHKdmVjYI8LaFUWXMRAgxkAKCQB+VtpGSlm+zuyRRWmi3h+k9NqQCgyfmD cpPyQcfxo9hcSI0WaDFmpWg=
=Oa4v
-----END PGP SIGNATURE-----
Re: [PATCH 11/16] turn priviled operation into a macro in head_64.S
In reply to Glauber de Oliveira CostaQuoted 5 lines · Glauber de Oliveira Costa
under paravirt, read cr2 cannot be issued directly anymore. So wrap it in a macro, defined to the operation itself in case paravirt is off, but to something else if we have paravirt in the game
Is this actually needed? It's only used in the early fault handler in head_64.S. Will we be taking that path in the paravirt case? If so, should we disable the fault handler altogether, since the hypervisor can probably provide better diagnositcs.
J
Quoted 35 lines
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/kernel/head_64.S | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-)arch/x86/kernel/head_64.S +8
-1@@ -19,6 +19,13 @@ #include <asm/msr.h> #include <asm/cache.h> +#ifdef CONFIG_PARAVIRT +#include <asm/asm-offsets.h> +#include <asm/paravirt.h> +#else +#define GET_CR2_INTO_RCX movq %cr2, %rcx +#endif + /* we are not able to switch in one step to the final KERNEL ADRESS SPACE * because we need identity-mapped pages. *@@ -267,7 +274,7 @@ ENTRY(early_idt_handler) xorl %eax,%eax movq 8(%rsp),%rsi # get rip movq (%rsp),%rdx - movq %cr2,%rcx + GET_CR2_INTO_RCX leaq early_idt_msg(%rip),%rdi call early_printk cmpl $2,early_recursion_flag(%rip)
PATCH 11/16 turn priviled operation into a macro in head_64.S
In reply to Glauber de Oliveira Costaunder paravirt, read cr2 cannot be issued directly anymore. So wrap it in a macro, defined to the operation itself in case paravirt is off, but to something else if we have paravirt in the game
--- arch/x86/kernel/head_64.S | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-)
arch/x86/kernel/head_64.S +8 -1
@@ -19,6 +19,13 @@ #include <asm/msr.h> #include <asm/cache.h> +#ifdef CONFIG_PARAVIRT +#include <asm/asm-offsets.h> +#include <asm/paravirt.h> +#else +#define GET_CR2_INTO_RCX movq %cr2, %rcx +#endif + /* we are not able to switch in one step to the final KERNEL ADRESS SPACE * because we need identity-mapped pages. *
@@ -267,7 +274,7 @@ ENTRY(early_idt_handler) xorl %eax,%eax movq 8(%rsp),%rsi # get rip movq (%rsp),%rdx - movq %cr2,%rcx + GET_CR2_INTO_RCX leaq early_idt_msg(%rip),%rdi call early_printk cmpl $2,early_recursion_flag(%rip)
Signature · 1 line
1.4.4.2
PATCH 10/16 export cpu_gdt_descr
In reply to Glauber de Oliveira CostaWith paravirualization, hypervisors needs to handle the gdt, that was right to this point only used at very early inialization code. Hypervisors (lguest being the current case) are commonly modules, so make it an export
--- arch/x86/kernel/x8664_ksyms_64.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-)
arch/x86/kernel/x8664_ksyms_64.c +6 -0
@@ -8,6 +8,7 @@#include <asm/processor.h> #include <asm/uaccess.h> #include <asm/pgtable.h> +#include <asm/desc.h> EXPORT_SYMBOL (kernel_thread);
@@ -60,3 +61,8 @@ EXPORT_SYMBOL(init_level4_pgt);EXPORT_SYMBOL (load_gs_index);EXPORT_SYMBOL (_proxy_pda); + +#ifdef CONFIG_PARAVIRT +/* Virtualized guests may want to use it */ +EXPORT_SYMBOL_GPL (cpu_gdt_descr); +#endif
Signature · 1 line
1.4.4.2
PATCH 9/16 This patch add provisions for time related functions so they
In reply to Glauber de Oliveira Costacan be later replaced by paravirt versions.
it basically encloses {g,s}et_wallclock inside the already existent functions update_persistent_clock and read_persistent_clock, and defines {s,g}et_wallclock to the core of such functions.
it also allow for a later-on-game time initialization, as done by i386. Paravirt guests can set a function to do their own initialization this way.
--- arch/x86/kernel/time_64.c | 42 ++++++++++++++++++++++++++++-------------- include/asm-x86/time.h | 26 ++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 18 deletions(-)
arch/x86/kernel/time_64.c +28 -14
@@ -33,9 +33,12 @@#include <acpi/acpi_bus.h> #endif #include <asm/i8253.h> +#include <asm/arch_hooks.h> #include <asm/pgtable.h> #include <asm/vsyscall.h> +#include <asm/time.h> #include <asm/timex.h> +#include <asm/timer.h> #include <asm/proto.h> #include <asm/hpet.h> #include <asm/sections.h>
@@ -77,18 +80,12 @@ EXPORT_SYMBOL(profile_pc);* sheet for details. */ -static int set_rtc_mmss (unsigned long nowtime) +int do_set_rtc_mmss (unsigned long nowtime) {int retval =0 ;int real_seconds, real_minutes, cmos_minutes;unsigned char control, freq_select; -/* -* IRQs are disabled when we're called from the timer interrupt, -* no need for spin_lock_irqsave() -*/ - -spin_lock (&rtc_lock);/* * Tell the clock it's being set and stop it.
@@ -138,14 +135,22 @@ static int set_rtc_mmss(unsigned long nowtime)CMOS_WRITE (control,RTC_CONTROL );CMOS_WRITE (freq_select,RTC_FREQ_SELECT ); -spin_unlock (&rtc_lock); -return retval; }int update_persistent_clock (struct timespec now) { -return set_rtc_mmss (now.tv_sec); +int retval; + +/* +* IRQs are disabled when we're called from the timer interrupt, +* no need for spin_lock_irqsave() +*/ +spin_lock (&rtc_lock); + retval =set_wallclock (now.tv_sec); +spin_unlock (&rtc_lock); + +return retval; }static irqreturn_t timer_event_interrupt (int irq,void *dev_id)
@@ -157,7 +162,7 @@ static irqreturn_t timer_event_interrupt(int irq, void *dev_id)return IRQ_HANDLED ; } -unsigned long read_persistent_clock (void ) +unsigned long do_get_cmos_time (void ) {unsigned int year, mon, day, hour, min, sec;unsigned long flags;
@@ -208,10 +213,15 @@ unsigned long read_persistent_clock(void)return mktime (year, mon, day, hour, min, sec); } +unsigned long read_persistent_clock (void ) +{ +return get_wallclock (); +} +/* calibrate_cpu is used on systems with fixed rate TSCs to determine * processor frequency */ #define TICK_COUNT 100000000 -static unsigned int __init tsc_calibrate_cpu_khz (void ) +unsigned long __init native_calculate_cpu_khz (void ) {int tsc_start, tsc_now;int i, no_ctr_free;
@@ -261,20 +271,23 @@ static struct irqaction irq0 = { .name = "timer" }; -void __init time_init (void ) +void __init hpet_time_init (void ) {if (!hpet_enable ())setup_pit_timer ();setup_irq (0 , &irq0); +} +void __init time_init (void ) +{tsc_calibrate (); cpu_khz = tsc_khz;if (cpu_has (&boot_cpu_data,X86_FEATURE_CONSTANT_TSC ) && boot_cpu_data.x86_vendor ==X86_VENDOR_AMD && boot_cpu_data.x86 ==16 ) - cpu_khz =tsc_calibrate_cpu_khz (); + cpu_khz =calculate_cpu_khz ();if (unsynchronized_tsc ())mark_tsc_unstable ("TSCs unsynchronized" );
@@ -287,4 +300,5 @@ void __init time_init(void)printk (KERN_INFO "time.c: Detected %d.%03d MHz processor.\n" , cpu_khz /1000 , cpu_khz %1000 );init_tsc_clocksource (); + late_time_init =choose_time_init (); }
include/asm-x86/time.h +22 -4
@@ -1,6 +1,10 @@ -#ifndef _ASMi386_TIME_H -#define _ASMi386_TIME_H +#ifndef _ASMX86_TIME_H +#define _ASMX86_TIME_H +extern void (*late_time_init)(void ); +extern void hpet_time_init (void ); + +#ifndef CONFIG_X86_64 #include <linux/efi.h> #include "mach_time.h"
@@ -28,8 +32,22 @@ static inline int native_set_wallclock(unsigned long nowtime) return retval; } -extern void (*late_time_init)(void ); -extern void hpet_time_init (void ); +#else +extern unsigned long do_get_cmos_time (void ); +extern int do_set_rtc_mmss (unsigned long nowtime); +extern void native_time_init_hook (void ); + +static inline unsigned long native_get_wallclock (void ) +{ +return do_get_cmos_time (); +} + +static inline int native_set_wallclock (unsigned long nowtime) +{ +return do_set_rtc_mmss (nowtime); +} + +#endif #ifdef CONFIG_PARAVIRT #include <asm/paravirt.h>
Signature · 1 line
1.4.4.2
PATCH 8/16 add native functions for descriptors handling
In reply to Glauber de Oliveira CostaThis patch turns the basic descriptor handling into native_ functions. It is basically write_idt, load_idt, write_gdt, load_gdt, set_ldt, store_tr, load_tls, and the ones for updating a single entry.
In the process of doing that, we change the definition of load_LDT_nolock, and caller sites have to be patched. We also patch call sites that now needs a typecast.
--- arch/x86/kernel/ldt_64.c | 7 +- include/asm-x86/desc_64.h | 172 +++++++++++++++++++++++++------------- include/asm-x86/mmu_context_64.h | 23 ++++-- 3 files changed, 132 insertions(+), 70 deletions(-)
arch/x86/kernel/ldt_64.c +3 -4
@@ -171,7 +171,7 @@ static int write_ldt(void __user * ptr, unsigned long bytecount, int oldmode) {struct task_struct *me = current;struct mm_struct * mm = me->mm; -__u32 entry_1, entry_2, *lp; +__u32 entry_1, entry_2;int error;struct user_desc ldt_info;
@@ -200,7 +200,6 @@ static int write_ldt(void __user * ptr, unsigned long bytecount, int oldmode) goto out_unlock; } - lp = (__u32 *) ((ldt_info.entry_number <<3 ) + (char *) mm->context.ldt);/* Allow LDTs to be cleared by the user. */ if (ldt_info.base_addr ==0 && ldt_info.limit ==0 ) {
@@ -218,8 +217,8 @@ static int write_ldt(void __user * ptr, unsigned long bytecount, int oldmode)/* Install the new entry ... */ install: - *lp = entry_1; - *(lp+1 ) = entry_2; +write_ldt_entry (mm->context.ldt, ldt_info.entry_number, + entry_1, entry_2); error =0 ; out_unlock:
include/asm-x86/desc_64.h +112 -60
236 lines not on this page · raw message
include/asm-x86/mmu_context_64.h +17 -6
@@ -7,7 +7,16 @@#include <asm/pda.h> #include <asm/pgtable.h> #include <asm/tlbflush.h> + +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #include <asm-generic/mm_hooks.h> +static inline void paravirt_activate_mm (struct mm_struct *prev, +struct mm_struct *next) +{ +} +#endif /* CONFIG_PARAVIRT */ /* * possibly do the LDT unload here?
@@ -25,7 +34,7 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)static inline void load_cr3 (pgd_t *pgd) { - asmvolatile ("movq %0,%%cr3" ::"r" (__pa (pgd)) :"memory" ); +write_cr3 (__pa (pgd)); } static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
@@ -43,7 +52,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,load_cr3 (next->pgd);if (unlikely (next->context.ldt != prev->context.ldt)) -load_LDT_nolock (&next->context, cpu); +load_LDT_nolock (&next->context); }#ifdef CONFIG_SMP else {
@@ -56,7 +65,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,* to make sure to use no freed page tables. */ load_cr3 (next->pgd); - load_LDT_nolock(&next->context, cpu); + load_LDT_nolock(&next->context); } }#endif
@@ -67,8 +76,10 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, asmvolatile ("movl %0,%%fs" ::"r" (0 )); \ } while(0) -#define activate_mm (prev, next) \ - switch_mm((prev),(next),NULL) - +#define activate_mm (prev, next) \ +do { \ + paravirt_activate_mm(prev, next); \ + switch_mm((prev), (next), NULL); \ +} while (0)#endif
Signature · 1 line
1.4.4.2
PATCH 7/16 native versions for set pagetables
In reply to Glauber de Oliveira CostaThis patch turns the set_p{te,md,ud,gd} functions into their native_ versions. There is no need to patch any caller.
Also, it adds pte_update() and pte_update_defer() calls whenever we modify a page table entry. This last part was coded to match i386 as close as possible.
Pieces of the header are moved to below the #ifdef CONFIG_PARAVIRT site, as they are users of the newly defined set_* macros.
--- include/asm-x86/pgtable_64.h | 192 ++++++++++++++++++++++++++++-------------- 1 files changed, 128 insertions(+), 64 deletions(-)
include/asm-x86/pgtable_64.h +128 -64
@@ -57,56 +57,107 @@ extern unsigned long empty_zero_page[PAGE_SIZE/sizeof(unsigned long)];*/ #define PTRS_PER_PTE 512 -#ifndef __ASSEMBLY__ +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else + +#define set_pte native_set_pte +#define set_pte_at (mm, addr, ptep, pteval) set_pte(ptep, pteval) +#define set_pmd native_set_pmd +#define set_pud native_set_pud +#define set_pgd native_set_pgd +#define pte_clear (mm, addr, xp) \ +do { \ + set_pte_at(mm, addr, xp, __pte(0)); \ +} while (0) -#define pte_ERROR (e) \ - printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), pte_val(e)) -#define pmd_ERROR (e) \ - printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), pmd_val(e)) -#define pud_ERROR (e) \ - printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), pud_val(e)) -#define pgd_ERROR (e) \ - printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e)) +#define pmd_clear (xp) do { set_pmd(xp, __pmd(0)); } while (0) +#define pud_clear native_pud_clear +#define pgd_clear native_pgd_clear +#define pte_update (mm, addr, ptep) do { } while (0) +#define pte_update_defer (mm, addr, ptep) do { } while (0) -#define pgd_none (x) (!pgd_val(x)) -#define pud_none (x) (!pud_val(x)) +#endif -static inline void set_pte (pte_t *dst,pte_t val) +#ifndef __ASSEMBLY__ + +static inline void native_set_pte (pte_t *dst,pte_t val) { -pte_val (*dst) =pte_val (val); + dst->pte =pte_val (val); } -#define set_pte_at (mm,addr,ptep,pteval) set_pte(ptep,pteval) -static inline void set_pmd (pmd_t *dst,pmd_t val) +static inline void native_set_pmd (pmd_t *dst,pmd_t val) { -pmd_val (*dst) =pmd_val (val); + dst->pmd =pmd_val (val); } -static inline void set_pud (pud_t *dst,pud_t val) +static inline void native_set_pud (pud_t *dst,pud_t val) { -pud_val (*dst) =pud_val (val); + dst->pud =pud_val (val); } -static inline void pud_clear (pud_t *pud) +static inline void native_set_pgd (pgd_t *dst,pgd_t val) { -set_pud (pud,__pud (0 )); + dst->pgd =pgd_val (val); } -static inline void set_pgd (pgd_t *dst,pgd_t val) +static inline void native_pud_clear (pud_t *pud) { -pgd_val (*dst) =pgd_val (val); -} +set_pud (pud,__pud (0 )); +} -static inline void pgd_clear (pgd_t * pgd) +static inline void native_pgd_clear (pgd_t *pgd) {set_pgd (pgd,__pgd (0 )); } -#define ptep_get_and_clear (mm,addr,xp) __pte(xchg(&(xp)->pte, 0)) +static inline void native_set_pte_at (struct mm_struct *mm,unsigned long addr, +pte_t *ptep,pte_t pteval) +{ +native_set_pte (ptep, pteval); +} + +static inline void native_pte_clear (struct mm_struct *mm,unsigned long addr, +pte_t *ptep) +{ +native_set_pte_at (mm, addr, ptep,__pte (0 )); +} + +static inline void native_pmd_clear (pmd_t *pmd) +{ +native_set_pmd (pmd,__pmd (0 )); +} + + +#define pte_ERROR (e) \ + printk("%s:%d: bad pte %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pte_val(e)) +#define pmd_ERROR (e) \ + printk("%s:%d: bad pmd %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pmd_val(e)) +#define pud_ERROR (e) \ + printk("%s:%d: bad pud %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pud_val(e)) +#define pgd_ERROR (e) \ + printk("%s:%d: bad pgd %p(%016llx).\n", \ + __FILE__, __LINE__, &(e), (u64)pgd_val(e)) + +#define pgd_none (x) (!pgd_val(x)) +#define pud_none (x) (!pud_val(x))struct mm_struct ; -static inline pte_t ptep_get_and_clear_full (struct mm_struct *mm,unsigned long addr,pte_t *ptep,int full) +static inline pte_t ptep_get_and_clear (struct mm_struct *mm, +unsigned long addr,pte_t *ptep) +{ +pte_t pte =__pte (xchg (&ptep->pte,0 )); +pte_update (mm, addr, ptep); +return pte; +} + +static inline pte_t ptep_get_and_clear_full (struct mm_struct *mm, +unsigned long addr,pte_t *ptep, +int full) {pte_t pte;if (full) {
@@ -246,7 +297,6 @@ static inline unsigned long pmd_bad(pmd_t pmd)#define pte_none (x) (!pte_val(x))#define pte_present (x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE)) -#define pte_clear (mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)#define pages_to_mb (x) ((x) >> (20-PAGE_SHIFT))/* FIXME: is this right? */
@@ -255,11 +305,11 @@ static inline unsigned long pmd_bad(pmd_t pmd)static inline pte_t pfn_pte (unsigned long page_nr,pgprot_t pgprot) { -pte_t pte; -pte_val (pte) = (page_nr <<PAGE_SHIFT ); -pte_val (pte) |=pgprot_val (pgprot); -pte_val (pte) &= __supported_pte_mask; -return pte; +unsigned long pte; + pte = (page_nr <<PAGE_SHIFT ); + pte |=pgprot_val (pgprot); + pte &= __supported_pte_mask; +return __pte (pte); }/*
@@ -283,30 +333,6 @@ static inline pte_t pte_mkwrite(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) |static inline pte_t pte_mkhuge (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) | _PAGE_PSE));return pte; }static inline pte_t pte_clrhuge (pte_t pte) {set_pte (&pte,__pte (pte_val (pte) & ~_PAGE_PSE));return pte; } -struct vm_area_struct ; - -static inline int ptep_test_and_clear_young (struct vm_area_struct *vma,unsigned long addr,pte_t *ptep) -{ -if (!pte_young (*ptep)) -return 0 ; -return test_and_clear_bit (_PAGE_BIT_ACCESSED, &ptep->pte); -} - -static inline void ptep_set_wrprotect (struct mm_struct *mm,unsigned long addr,pte_t *ptep) -{ -clear_bit (_PAGE_BIT_RW, &ptep->pte); -} - -/* -* Macro to mark a page protection value as "uncacheable". -*/ -#define pgprot_noncached (prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) - -static inline int pmd_large (pmd_t pte) { -return (pmd_val (pte) & __LARGE_PTE) == __LARGE_PTE; -} - -/* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to.
@@ -340,7 +366,6 @@ static inline int pmd_large(pmd_t pte) { pmd_index(address))#define pmd_none (x) (!pmd_val(x))#define pmd_present (x) (pmd_val(x) & _PAGE_PRESENT) -#define pmd_clear (xp) do { set_pmd(xp, __pmd(0)); } while (0)#define pfn_pmd (nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot)))#define pmd_pfn (x) ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT)
@@ -352,15 +377,53 @@ static inline int pmd_large(pmd_t pte) {/* page, protection -> pte */ #define mk_pte (page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) -#define mk_pte_huge (entry) (pte_val(entry) |= _PAGE_PRESENT | _PAGE_PSE) - + +static inline pte_t __mk_pte_huge (pte_t entry) +{ +unsigned long pte; + pte =pte_val (entry); + pte |= _PAGE_PRESENT | _PAGE_PSE; +return __pte (pte); +} +#define mk_pte_huge (entry) ((entry) = __mk_pte_huge(entry)) + +#include <linux/mm_types.h> +static inline int ptep_test_and_clear_young (struct vm_area_struct *vma, +unsigned long addr,pte_t *ptep) +{ +int ret =0 ; +if (!pte_young (*ptep)) +return 0 ; + ret =test_and_clear_bit (_PAGE_BIT_ACCESSED, &ptep->pte); +pte_update (vma->vm_mm, addr, ptep); +return ret; +} + +static inline void ptep_set_wrprotect (struct mm_struct *mm, +unsigned long addr,pte_t *ptep) +{ +clear_bit (_PAGE_BIT_RW, &ptep->pte); +pte_update (mm, addr, ptep); +} + +/* +* Macro to mark a page protection value as "uncacheable". +*/ +#define pgprot_noncached (prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) + +static inline int pmd_large (pmd_t pte) +{ +return (pmd_val (pte) & __LARGE_PTE) == __LARGE_PTE; +} +/* Change flags of a PTE */ -static inline pte_t pte_modify (pte_t pte,pgprot_t newprot) +static inline pte_t pte_modify (pte_t pte_old,pgprot_t newprot) { -pte_val (pte) &= _PAGE_CHG_MASK; -pte_val (pte) |=pgprot_val (newprot); -pte_val (pte) &= __supported_pte_mask; -return pte; +unsigned long pte =pte_val (pte_old); + pte &= _PAGE_CHG_MASK; + pte |=pgprot_val (newprot); + pte &= __supported_pte_mask; +return __pte (pte); }#define pte_index (address) \
@@ -387,6 +450,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)int __changed = !pte_same (*(__ptep), __entry); \if (__changed && __dirty) { \set_pte (__ptep, __entry); \ +pte_update_defer ((__vma)->vm_mm, (__address), (__ptep)); \flush_tlb_page (__vma, __address); \ } \ __changed; \
Signature · 1 line
1.4.4.2
PATCH 6/16 export math_state_restore
In reply to Glauber de Oliveira CostaExport math_state_restore symbol, so it can be used for hypervisors. They are commonly loaded as modules (lguest being an example).
--- arch/x86/kernel/traps_64.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)
arch/x86/kernel/traps_64.c +1 -0
@@ -1077,6 +1077,7 @@ asmlinkage void math_state_restore(void)task_thread_info (me)->status |=TS_USEDFPU ; me->fpu_counter++; } +EXPORT_SYMBOL_GPL (math_state_restore);void __init trap_init (void ) {
Signature · 1 line
1.4.4.2
PATCH 5/16 report ring kernel is running without paravirt
In reply to Glauber de Oliveira CostaWhen paravirtualization is disabled, the kernel is always running at ring 0. So report it in the appropriate macro
--- include/asm-x86/segment_64.h | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-)
include/asm-x86/segment_64.h +4 -0
@@ -50,4 +50,8 @@#define GDT_SIZE (GDT_ENTRIES * 8)#define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES * 8) +#ifndef CONFIG_PARAVIRT +#define get_kernel_rpl () 0 +#endif +#endif
Signature · 1 line
1.4.4.2
PATCH 4/16 provide native irq initialization function
In reply to Glauber de Oliveira CostaThe interrupt initialization routine becomes native_init_IRQ and will be overriden later in case paravirt is on. The interrupt array is made visible for guests such lguest, that will need to have their own initialization mechanism (though using most of the same irq lines) later on.
--- arch/x86/kernel/i8259_64.c | 7 +++++-- include/asm-x86/irq_64.h | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-)
arch/x86/kernel/i8259_64.c +5 -2
@@ -76,7 +76,7 @@ BUILD_16_IRQS(0xc) BUILD_16_IRQS(0xd) BUILD_16_IRQS(0xe) BUILD_16_IRQS(0xf)IRQ (x,c),IRQ (x,d),IRQ (x,e),IRQ (x,f) /* for the irq vectors */ -static void (*interrupt[NR_VECTORS -FIRST_EXTERNAL_VECTOR ])(void ) = { +void (*interrupt[NR_VECTORS -FIRST_EXTERNAL_VECTOR ])(void ) = {IRQLIST_16 (0x2 ),IRQLIST_16 (0x3 ),IRQLIST_16 (0x4 ),IRQLIST_16 (0x5 ),IRQLIST_16 (0x6 ),IRQLIST_16 (0x7 ),IRQLIST_16 (0x8 ),IRQLIST_16 (0x9 ),IRQLIST_16 (0xa ),IRQLIST_16 (0xb ),
@@ -448,7 +448,10 @@ void __init init_ISA_irqs (void) } } -void __init init_IRQ (void ) +/* Overridden in paravirt.c */ +void init_IRQ (void ) __attribute__((weak,alias ("native_init_IRQ" ))); + +void __init native_init_IRQ (void ) {int i;
include/asm-x86/irq_64.h +3 -0
@@ -46,6 +46,9 @@ static __inline__ int irq_canonicalize(int irq)extern void fixup_irqs (cpumask_t map);#endif +#include <linux/init.h> +void native_init_IRQ (void ); +#define __ARCH_HAS_DO_SOFTIRQ 1#endif /* _ASM_IRQ_H */
Signature · 1 line
1.4.4.2
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Glauber de Oliveira CostaQuoted 45 lines · Glauber de Oliveira Costa
This patch introduces, and patch callers when needed, native versions for read/write_crX functions, clts and wbinvd.
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/mm/pageattr_64.c | 3 +- include/asm-x86/system_64.h | 60 ++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 18 deletions(-)arch/x86/mm/pageattr_64.c +2
-1@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/tlbflush.h> #include <asm/io.h> +#include <asm/paravirt.h> pte_t *lookup_address (unsigned long address) {@@ -77,7 +78,7 @@ static void flush_kernel_map(void *arg)much cheaper than WBINVD. */ /* clflush is still broken. Disable for now. */ if (1 || !cpu_has_clflush) - asmvolatile ("wbinvd" :::"memory" ); +wbinvd (); else list_for_each_entry(pg, l, lru) {void *adr =page_address (pg);clflush_cache_range (adr,PAGE_SIZE );include/asm-x86/system_64.h +4
-1@@ -65,53 +65,62 @@ extern void load_gs_index(unsigned);/* * Clear and set 'TS' bit respectively */ -#define clts () __asm__ __volatile__ ("clts") +static inline void native_clts (void ) +{ + asmvolatile ("clts" ); +}
include/asm-x86/system_64.h extern void load_gs_index(unsigned); · line 79
Quoted 10 lines
include/asm-x86/system_64.h +2
-2-static inline unsigned long read_cr0 (void ) -{ +static inline unsigned long native_read_cr0 (void ) +{unsigned long cr0; asmvolatile ("movq %%cr0,%0" :"=r" (cr0));return cr0; }
This is a pre-existing bug, but it seems to me that these read/write crX asms should have a constraint to stop the compiler from reordering them with respect to each other. The brute-force approach would be to add "memory" clobbers, but the subtle fix would be to add a variable which is only used to sequence:
static int __cr_seq;
static inline unsigned long native_read_cr0(void)
{
unsigned long cr0;
asm volatile("mov %%cr0, %0" : "=r" (cr0), "=m" (__cr_seq));
}static inline void native_write_cr0(unsigned long val)
{
asm volatile("mov %1, %%cr0" : "+m" (__cr_seq) : "r" (val));
}J
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Jeremy Fitzhardinge escreveu:
Quoted 47 lines
Quoted 46 lines · Glauber de Oliveira Costa
This patch introduces, and patch callers when needed, native versions for read/write_crX functions, clts and wbinvd.
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/mm/pageattr_64.c | 3 +- include/asm-x86/system_64.h | 60 ++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 18 deletions(-)arch/x86/mm/pageattr_64.c +2
-1@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/tlbflush.h> #include <asm/io.h> +#include <asm/paravirt.h> pte_t *lookup_address (unsigned long address) {@@ -77,7 +78,7 @@ static void flush_kernel_map(void *arg)much cheaper than WBINVD. */ /* clflush is still broken. Disable for now. */ if (1 || !cpu_has_clflush) - asmvolatile ("wbinvd" :::"memory" ); +wbinvd (); else list_for_each_entry(pg, l, lru) {void *adr =page_address (pg);clflush_cache_range (adr,PAGE_SIZE );include/asm-x86/system_64.h +4
-1@@ -65,53 +65,62 @@ extern void load_gs_index(unsigned);/* * Clear and set 'TS' bit respectively */ -#define clts () __asm__ __volatile__ ("clts") +static inline void native_clts (void ) +{ + asmvolatile ("clts" ); +}
Quoted 16 lines
include/asm-x86/system_64.h extern void load_gs_index(unsigned); · line 79
Quoted 9 lines
include/asm-x86/system_64.h +2
-2-static inline unsigned long read_cr0 (void ) -{ +static inline unsigned long native_read_cr0 (void ) +{unsigned long cr0; asmvolatile ("movq %%cr0,%0" :"=r" (cr0));return cr0; }This is a pre-existing bug, but it seems to me that these read/write crX asms should have a constraint to stop the compiler from reordering them with respect to each other. The brute-force approach would be to add "memory" clobbers, but the subtle fix would be to add a variable which is only used to sequence:
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it. Yet, I agree with you that the theorectical problem exists for the reorder, and your proposed fix seems fine (although if we're really desperate about memory usage, we can use a char instead a int and save 3 bytes!)
It also just ocurred to me that this part of the patch can also go into the consolidation part. So I'll respin it.
Thanks for the comment
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.7 (GNU/Linux)
Comment: Using GnuPG with Remi - http://enigmail.mozdev.org
iD8DBQFHKdkujYI8LaFUWXMRAhV2AKDPIjwGQnoLtldys/OWtIEs6biwxwCg1Jd/ o36S+qcb4sWJ6peqhrSRnos=
=YmeC
-----END PGP SIGNATURE-----
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
Quoted 3 lines · Glauber de Oliveira Costa
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it.
Well, volatile will make a read be repeated rather than caching the previous value, but it has no effect on ordering.
Quoted 4 lines
Yet, I agree with you that
the theorectical problem exists for the reorder, and your proposed fix seems fine (although if we're really desperate about memory usage, we can use a char instead a int and save 3 bytes!)
Sure. Ideally the compiler would never even generate a reference to it, and it could just be extern, but in practice the compiler will generate references sometimes.
J
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
Quoted 7 lines · Jeremy Fitzhardinge
Quoted 3 lines · Glauber de Oliveira Costa
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it.Well, volatile will make a read be repeated rather than caching the previous value, but it has no effect on ordering.
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
-- Keir
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Keir Fraser escreveu:
Quoted 12 lines · Jeremy Fitzhardinge
Quoted 7 lines
Quoted 3 lines · Glauber de Oliveira Costa
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it.Well, volatile will make a read be repeated rather than caching the previous value, but it has no effect on ordering.
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
According to a gcc developer to whom I asked this question, volatile prevents the code
to be removed, but does not prevent it to be moved (pun indented). In practice, it should force
a re-read, but not influence the ordering decisions from the compiler. Besides , 'significantly'
sounds like a significantly unprecise word, whose specific meaning may be implementation dependant.
So I agree that adding a memory location reference is probably the best alternative.
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.7 (GNU/Linux)
Comment: Using GnuPG with Remi - http://enigmail.mozdev.org
iD8DBQFHKftDjYI8LaFUWXMRAiLTAKDqf/M8umNYw6u7r9ONozTEUVy8SwCgygma jWNKQmxmLpyPxr00KbQy9Vg=
=JM4K
-----END PGP SIGNATURE-----
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Keir Fraser-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Keir Fraser escreveu:
Quoted 12 lines · Jeremy Fitzhardinge
Quoted 7 lines
Quoted 3 lines · Glauber de Oliveira Costa
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it.Well, volatile will make a read be repeated rather than caching the previous value, but it has no effect on ordering.
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
According to a gcc developer to whom I asked this question, volatile prevents the code
to be removed, but does not prevent it to be moved (pun indented). In practice, it should force
a re-read, but not influence the ordering decisions from the compiler. Besides , 'significantly'
sounds like a significantly unprecise word, whose specific meaning may be implementation dependant.
So I agree that adding a memory location reference is probably the best alternative.
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.7 (GNU/Linux)
Comment: Using GnuPG with Remi - http://enigmail.mozdev.org
iD8DBQFHKftDjYI8LaFUWXMRAiLTAKDqf/M8umNYw6u7r9ONozTEUVy8SwCgygma jWNKQmxmLpyPxr00KbQy9Vg=
=JM4K
-----END PGP SIGNATURE-----
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Keir FraserQuoted 3 lines · Keir Fraser
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
That phrase doesn't appear in the gcc manual; in fact, it specifically says that reordering can happen:
The volatile' keyword indicates that the instruction has important side-effects. GCC will not delete a volatile asm' if it is reachable. (The instruction can still be deleted if GCC can prove that control-flow will never reach the location of the instruction.) Note that even a volatile `asm' instruction can be moved relative to other code, including across jump instructions. For example, on many targets there is a system register which can be set to control the rounding mode of floating point operations. You might try setting it with a volatile `asm', like this PowerPC example:
asm volatile("mtfsf 255,%0" : : "f" (fpenv)); sum = x + y;
This will not work reliably, as the compiler may move the addition back before the volatile asm'. To make it work you need to add an artificial dependency to the asm' referencing a variable in the code you don't want moved, for example:
asm volatile ("mtfsf 255,%1" : "=X"(sum): "f"(fpenv)); sum = x + y;
I take from this that it is not a good idea to assume "asm volatile" has any ordering effects at all.
J
Re: [Lguest] [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
Quoted 4 lines · Jeremy Fitzhardinge
Quoted 3 lines · Keir Fraser
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
I understood it as reordering was permitted, but no re-ordering across another volatile load, store, or asm was permitted. And of course, as long as input and output constraints are written properly, the re-ordering should not be vulnerable to pathological movement causing the code to malfunction.
It seems that CPU state side effects which can't be expressed in C need special care - FPU is certainly one example.
Also, memory clobber on a volatile asm should stop invalid movement across TLB flushes and other problems areas. Even memory fences should have memory clobber in order to stop movement of loads and stores across the fence by the compiler.
Zach
Re: [Lguest] [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
Quoted 2 lines · Zachary Amsden
I understood it as reordering was permitted, but no re-ordering across another volatile load, store, or asm was permitted.
It doesn't say that, so I wouldn't assume it. Certainly we had problems with the pda code; until I added the _proxy_pda dependency variable, the only fix Andi could find was adding both "volatile" and a memory clobber.
Quoted 5 lines
And of course, as
long as input and output constraints are written properly, the re-ordering should not be vulnerable to pathological movement causing the code to malfunction.
Yes. I think constraints are the only way to control ordering (even if it's as heavy-handed as a memory clobber). It would be nice if gcc had a constraint which was only used for ordering, and never generated a reference. Then you could make up pseudo-variables in order to express dependencies without having the risk that the compiler would generate references.
Quoted 3 lines
It seems that CPU state side effects which can't be expressed in C need special care - FPU is certainly one example.
Not an immediate problem, fortunately.
Quoted 2 lines
Also, memory clobber on a volatile asm should stop invalid movement across TLB flushes and other problems areas.
Yes. Any asm which has global effects on how addresses are interpreted (like tlbflush, reloading the pagetable base, changing modes, etc) needs to have a memory clobber.
Quoted 4 lines
Even memory fences should
have memory clobber in order to stop movement of loads and stores across the fence by the compiler.
Pretty sure they do. A normal compiler barrier is *just* a memory clobber.
J
-------------------------------------------------------------------------
This SF.net email is sponsored by: Splunk Inc.
Still grepping through log files to find problems? Stop. Now Search log events and configuration files using AJAX and a browser. Download your FREE copy of Splunk now >> http://get.splunk.com/
Re: [Lguest] [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Zachary AmsdenQuoted 2 lines · Zachary Amsden
I understood it as reordering was permitted, but no re-ordering across another volatile load, store, or asm was permitted.
It doesn't say that, so I wouldn't assume it. Certainly we had problems with the pda code; until I added the _proxy_pda dependency variable, the only fix Andi could find was adding both "volatile" and a memory clobber.
Quoted 5 lines
And of course, as
long as input and output constraints are written properly, the re-ordering should not be vulnerable to pathological movement causing the code to malfunction.
Yes. I think constraints are the only way to control ordering (even if it's as heavy-handed as a memory clobber). It would be nice if gcc had a constraint which was only used for ordering, and never generated a reference. Then you could make up pseudo-variables in order to express dependencies without having the risk that the compiler would generate references.
Quoted 3 lines
It seems that CPU state side effects which can't be expressed in C need special care - FPU is certainly one example.
Not an immediate problem, fortunately.
Quoted 2 lines
Also, memory clobber on a volatile asm should stop invalid movement across TLB flushes and other problems areas.
Yes. Any asm which has global effects on how addresses are interpreted (like tlbflush, reloading the pagetable base, changing modes, etc) needs to have a memory clobber.
Quoted 4 lines
Even memory fences should
have memory clobber in order to stop movement of loads and stores across the fence by the compiler.
Pretty sure they do. A normal compiler barrier is *just* a memory clobber.
J
Re: [Lguest] [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Jeremy FitzhardingeQuoted 4 lines · Jeremy Fitzhardinge
Quoted 3 lines · Keir Fraser
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
I understood it as reordering was permitted, but no re-ordering across another volatile load, store, or asm was permitted. And of course, as long as input and output constraints are written properly, the re-ordering should not be vulnerable to pathological movement causing the code to malfunction.
It seems that CPU state side effects which can't be expressed in C need special care - FPU is certainly one example.
Also, memory clobber on a volatile asm should stop invalid movement across TLB flushes and other problems areas. Even memory fences should have memory clobber in order to stop movement of loads and stores across the fence by the compiler.
Zach
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Keir FraserQuoted 3 lines · Keir Fraser
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
That phrase doesn't appear in the gcc manual; in fact, it specifically says that reordering can happen:
The volatile' keyword indicates that the instruction has important side-effects. GCC will not delete a volatile asm' if it is reachable. (The instruction can still be deleted if GCC can prove that control-flow will never reach the location of the instruction.) Note that even a volatile `asm' instruction can be moved relative to other code, including across jump instructions. For example, on many targets there is a system register which can be set to control the rounding mode of floating point operations. You might try setting it with a volatile `asm', like this PowerPC example:
asm volatile("mtfsf 255,%0" : : "f" (fpenv)); sum = x + y;
This will not work reliably, as the compiler may move the addition back before the volatile asm'. To make it work you need to add an artificial dependency to the asm' referencing a variable in the code you don't want moved, for example:
asm volatile ("mtfsf 255,%1" : "=X"(sum): "f"(fpenv)); sum = x + y;
I take from this that it is not a good idea to assume "asm volatile" has any ordering effects at all.
J
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Jeremy FitzhardingeQuoted 7 lines · Jeremy Fitzhardinge
Quoted 3 lines · Glauber de Oliveira Costa
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it.Well, volatile will make a read be repeated rather than caching the previous value, but it has no effect on ordering.
volatile prevents the asm from being 'moved significantly', according to the gcc manual. I take that to mean that reordering is not allowed.
-- Keir
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Glauber de Oliveira CostaQuoted 3 lines · Glauber de Oliveira Costa
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it.
Well, volatile will make a read be repeated rather than caching the previous value, but it has no effect on ordering.
Quoted 4 lines
Yet, I agree with you that
the theorectical problem exists for the reorder, and your proposed fix seems fine (although if we're really desperate about memory usage, we can use a char instead a int and save 3 bytes!)
Sure. Ideally the compiler would never even generate a reference to it, and it could just be extern, but in practice the compiler will generate references sometimes.
J
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Jeremy Fitzhardinge-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
Jeremy Fitzhardinge escreveu:
Quoted 47 lines
Quoted 46 lines · Glauber de Oliveira Costa
This patch introduces, and patch callers when needed, native versions for read/write_crX functions, clts and wbinvd.
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/mm/pageattr_64.c | 3 +- include/asm-x86/system_64.h | 60 ++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 18 deletions(-)arch/x86/mm/pageattr_64.c +2
-1@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/tlbflush.h> #include <asm/io.h> +#include <asm/paravirt.h> pte_t *lookup_address (unsigned long address) {@@ -77,7 +78,7 @@ static void flush_kernel_map(void *arg)much cheaper than WBINVD. */ /* clflush is still broken. Disable for now. */ if (1 || !cpu_has_clflush) - asmvolatile ("wbinvd" :::"memory" ); +wbinvd (); else list_for_each_entry(pg, l, lru) {void *adr =page_address (pg);clflush_cache_range (adr,PAGE_SIZE );include/asm-x86/system_64.h +4
-1@@ -65,53 +65,62 @@ extern void load_gs_index(unsigned);/* * Clear and set 'TS' bit respectively */ -#define clts () __asm__ __volatile__ ("clts") +static inline void native_clts (void ) +{ + asmvolatile ("clts" ); +}
Quoted 16 lines
include/asm-x86/system_64.h extern void load_gs_index(unsigned); · line 79
Quoted 9 lines
include/asm-x86/system_64.h +2
-2-static inline unsigned long read_cr0 (void ) -{ +static inline unsigned long native_read_cr0 (void ) +{unsigned long cr0; asmvolatile ("movq %%cr0,%0" :"=r" (cr0));return cr0; }This is a pre-existing bug, but it seems to me that these read/write crX asms should have a constraint to stop the compiler from reordering them with respect to each other. The brute-force approach would be to add "memory" clobbers, but the subtle fix would be to add a variable which is only used to sequence:
I in fact have seen bugs with mixed reads and writes to the same cr, (cr4), but adding the volatile
flag to the read function seemed to fix it. Yet, I agree with you that the theorectical problem exists for the reorder, and your proposed fix seems fine (although if we're really desperate about memory usage, we can use a char instead a int and save 3 bytes!)
It also just ocurred to me that this part of the patch can also go into the consolidation part. So I'll respin it.
Thanks for the comment
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.7 (GNU/Linux)
Comment: Using GnuPG with Remi - http://enigmail.mozdev.org
iD8DBQFHKdkujYI8LaFUWXMRAhV2AKDPIjwGQnoLtldys/OWtIEs6biwxwCg1Jd/ o36S+qcb4sWJ6peqhrSRnos=
=YmeC
-----END PGP SIGNATURE-----
Re: [PATCH 3/16] read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Glauber de Oliveira CostaQuoted 45 lines · Glauber de Oliveira Costa
This patch introduces, and patch callers when needed, native versions for read/write_crX functions, clts and wbinvd.
Signed-off-by Glauber de Oliveira Costa <[email protected]>Signed-off-by Steven Rostedt <[email protected]>--- arch/x86/mm/pageattr_64.c | 3 +- include/asm-x86/system_64.h | 60 ++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 18 deletions(-)arch/x86/mm/pageattr_64.c +2
-1@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/tlbflush.h> #include <asm/io.h> +#include <asm/paravirt.h> pte_t *lookup_address (unsigned long address) {@@ -77,7 +78,7 @@ static void flush_kernel_map(void *arg)much cheaper than WBINVD. */ /* clflush is still broken. Disable for now. */ if (1 || !cpu_has_clflush) - asmvolatile ("wbinvd" :::"memory" ); +wbinvd (); else list_for_each_entry(pg, l, lru) {void *adr =page_address (pg);clflush_cache_range (adr,PAGE_SIZE );include/asm-x86/system_64.h +4
-1@@ -65,53 +65,62 @@ extern void load_gs_index(unsigned);/* * Clear and set 'TS' bit respectively */ -#define clts () __asm__ __volatile__ ("clts") +static inline void native_clts (void ) +{ + asmvolatile ("clts" ); +}
include/asm-x86/system_64.h extern void load_gs_index(unsigned); · line 79
Quoted 10 lines
include/asm-x86/system_64.h +2
-2-static inline unsigned long read_cr0 (void ) -{ +static inline unsigned long native_read_cr0 (void ) +{unsigned long cr0; asmvolatile ("movq %%cr0,%0" :"=r" (cr0));return cr0; }
This is a pre-existing bug, but it seems to me that these read/write crX asms should have a constraint to stop the compiler from reordering them with respect to each other. The brute-force approach would be to add "memory" clobbers, but the subtle fix would be to add a variable which is only used to sequence:
static int __cr_seq;
static inline unsigned long native_read_cr0(void)
{
unsigned long cr0;
asm volatile("mov %%cr0, %0" : "=r" (cr0), "=m" (__cr_seq));
}static inline void native_write_cr0(unsigned long val)
{
asm volatile("mov %1, %%cr0" : "+m" (__cr_seq) : "r" (val));
}J
PATCH 3/16 read/write_crX, clts and wbinvd for 64-bit paravirt
In reply to Glauber de Oliveira CostaThis patch introduces, and patch callers when needed, native versions for read/write_crX functions, clts and wbinvd.
--- arch/x86/mm/pageattr_64.c | 3 +- include/asm-x86/system_64.h | 60 ++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 18 deletions(-)
arch/x86/mm/pageattr_64.c +2 -1
@@ -12,6 +12,7 @@#include <asm/processor.h> #include <asm/tlbflush.h> #include <asm/io.h> +#include <asm/paravirt.h> pte_t *lookup_address (unsigned long address) {
@@ -77,7 +78,7 @@ static void flush_kernel_map(void *arg)much cheaper than WBINVD. */ /* clflush is still broken. Disable for now. */ if (1 || !cpu_has_clflush) - asmvolatile ("wbinvd" :::"memory" ); +wbinvd (); else list_for_each_entry(pg, l, lru) {void *adr =page_address (pg);clflush_cache_range (adr,PAGE_SIZE );
include/asm-x86/system_64.h +43 -17
106 lines not on this page · raw message
Signature · 1 line
1.4.4.2
PATCH 2/16 paravirt hooks at entry functions.
In reply to Glauber de Oliveira CostaThose are the hooks needed for paravirt at entry_64.S In general, they follow the way of i386.
--- arch/x86/kernel/entry_64.S | 108 +++++++++++++++++++++++++++----------------- 1 files changed, 66 insertions(+), 42 deletions(-)
arch/x86/kernel/entry_64.S +66 -42
@@ -50,6 +50,7 @@ #include <asm/hw_irq.h> #include <asm/page.h> #include <asm/irqflags.h> +#include <asm/paravirt.h> .code64
@@ -57,6 +58,20 @@ #define retint_kernel retint_restore_args #endif +#ifdef CONFIG_PARAVIRT +ENTRY(native_irq_enable_syscall_ret) + movq %gs:pda_oldrsp,%rsp + swapgs + sysretq +/* + * This could well be defined as a C function, but as it is only used here, + * let it be locally defined + */ +ENTRY(native_swapgs) + swapgs + retq +#endif /* CONFIG_PARAVIRT */ + .macro TRACE_IRQS_IRETQ offset=ARGOFFSET #ifdef CONFIG_TRACE_IRQFLAGS
@@ -216,14 +231,21 @@ ENTRY(system_call) CFI_DEF_CFA rsp,PDA_STACKOFFSET CFI_REGISTER rip,rcx /*CFI_REGISTER rflags,r11*/ - swapgs + SWAPGS_UNSAFE_STACK + /* + * A hypervisor implementation might want to use a label + * after the swapgs, so that it can do the swapgs + * for the guest and jump here on syscall. + */ +ENTRY(system_call_after_swapgs) + movq %rsp,%gs:pda_oldrsp movq %gs:pda_kernelstack,%rsp /* * No need to follow this irqs off/on section - it's straight * and short: */ - sti + ENABLE_INTERRUPTS(CLBR_NONE) SAVE_ARGS 8,1 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp)
@@ -246,7 +268,7 @@ ret_from_sys_call: sysret_check: LOCKDEP_SYS_EXIT GET_THREAD_INFO(%rcx) - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF movl threadinfo_flags(%rcx),%edx andl %edi,%edx
@@ -260,9 +282,7 @@ sysret_check: CFI_REGISTER rip,rcx RESTORE_ARGS 0,-ARG_SKIP,1 /*CFI_REGISTER rflags,r11*/ - movq %gs:pda_oldrsp,%rsp - swapgs - sysretq + ENABLE_INTERRUPTS_SYSCALL_RET CFI_RESTORE_STATE /* Handle reschedules */
@@ -271,7 +291,7 @@ sysret_careful: bt $TIF_NEED_RESCHED,%edx jnc sysret_signal TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) pushq %rdi CFI_ADJUST_CFA_OFFSET 8 call schedule
@@ -282,7 +302,7 @@ sysret_careful: /* Handle a signal */ sysret_signal: TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) testl $(_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY),%edx jz 1f
@@ -295,7 +315,7 @@ sysret_signal: 1: movl $_TIF_NEED_RESCHED,%edi /* Use IRET because user could have changed frame. This works because ptregscall_common has called FIXUP_TOP_OF_STACK. */ - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp int_with_check
@@ -327,7 +347,7 @@ tracesys: */ .globl int_ret_from_sys_call int_ret_from_sys_call: - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF testl $3,CS-ARGOFFSET(%rsp) je retint_restore_args
@@ -349,20 +369,20 @@ int_careful: bt $TIF_NEED_RESCHED,%edx jnc int_very_careful TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) pushq %rdi CFI_ADJUST_CFA_OFFSET 8 call schedule popq %rdi CFI_ADJUST_CFA_OFFSET -8 - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp int_with_check /* handle signals and tracing -- both require a full stack frame */ int_very_careful: TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) SAVE_REST /* Check for syscall exit trace */ testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
@@ -385,7 +405,7 @@ int_signal: 1: movl $_TIF_NEED_RESCHED,%edi int_restore_rest: RESTORE_REST - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp int_with_check CFI_ENDPROC
@@ -506,7 +526,7 @@ END(stub_rt_sigreturn) CFI_DEF_CFA_REGISTER rbp testl $3,CS(%rdi) je 1f - swapgs + SWAPGS /* irqcount is used to check if a CPU is already on an interrupt stack or not. While this is essentially redundant with preempt_count it is a little cheaper to use a separate counter in the PDA
@@ -527,7 +547,7 @@ ENTRY(common_interrupt) interrupt do_IRQ /* 0(%rsp): oldrsp-ARGOFFSET */ ret_from_intr: - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF decl %gs:pda_irqcount leaveq
@@ -556,13 +576,13 @@ retint_swapgs: /* return to user-space */ /* * The iretq could re-enable interrupts: */ - cli + DISABLE_INTERRUPTS(CLBR_ANY) TRACE_IRQS_IRETQ - swapgs + SWAPGS jmp restore_args retint_restore_args: /* return to kernel space */ - cli + DISABLE_INTERRUPTS(CLBR_ANY) /* * The iretq could re-enable interrupts: */
@@ -570,10 +590,14 @@ retint_restore_args: /* return to kernel space */ restore_args: RESTORE_ARGS 0,8,0 iret_label: +#ifdef CONFIG_PARAVIRT + INTERRUPT_RETURN +#endif +ENTRY(native_iret) iretq .section __ex_table,"a" - .quad iret_label,bad_iret + .quad native_iret, bad_iret .previous .section .fixup,"ax" /* force a signal here? this matches i386 behaviour */
@@ -581,24 +605,24 @@ iret_label: bad_iret: movq $11,%rdi /* SIGSEGV */ TRACE_IRQS_ON - sti - jmp do_exit - .previous - + ENABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI)) + jmp do_exit + .previous + /* edi: workmask, edx: work */ retint_careful: CFI_RESTORE_STATE bt $TIF_NEED_RESCHED,%edx jnc retint_signal TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) pushq %rdi CFI_ADJUST_CFA_OFFSET 8 call schedule popq %rdi CFI_ADJUST_CFA_OFFSET -8 GET_THREAD_INFO(%rcx) - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF jmp retint_check
@@ -606,14 +630,14 @@ retint_signal: testl $(_TIF_SIGPENDING|_TIF_SINGLESTEP|_TIF_MCE_NOTIFY),%edx jz retint_swapgs TRACE_IRQS_ON - sti + ENABLE_INTERRUPTS(CLBR_NONE) SAVE_REST movq $-1,ORIG_RAX(%rsp) xorl %esi,%esi # oldset movq %rsp,%rdi # &pt_regs call do_notify_resume RESTORE_REST - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF movl $_TIF_NEED_RESCHED,%edi GET_THREAD_INFO(%rcx)
@@ -731,7 +755,7 @@ END(spurious_interrupt) rdmsr testl %edx,%edx js 1f - swapgs + SWAPGS xorl %ebx,%ebx 1: .if \ist
@@ -747,7 +771,7 @@ END(spurious_interrupt) .if \ist addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) .endif - cli + DISABLE_INTERRUPTS(CLBR_NONE) .if \irqtrace TRACE_IRQS_OFF .endif
@@ -776,10 +800,10 @@ paranoid_swapgs\trace: .if \trace TRACE_IRQS_IRETQ 0 .endif - swapgs + SWAPGS_UNSAFE_STACK paranoid_restore\trace: RESTORE_ALL 8 - iretq + INTERRUPT_RETURN paranoid_userspace\trace: GET_THREAD_INFO(%rcx) movl threadinfo_flags(%rcx),%ebx
@@ -794,11 +818,11 @@ paranoid_userspace\trace: .if \trace TRACE_IRQS_ON .endif - sti + ENABLE_INTERRUPTS(CLBR_NONE) xorl %esi,%esi /* arg2: oldset */ movq %rsp,%rdi /* arg1: &pt_regs */ call do_notify_resume - cli + DISABLE_INTERRUPTS(CLBR_NONE) .if \trace TRACE_IRQS_OFF .endif
@@ -807,9 +831,9 @@ paranoid_schedule\trace: .if \trace TRACE_IRQS_ON .endif - sti + ENABLE_INTERRUPTS(CLBR_ANY) call schedule - cli + DISABLE_INTERRUPTS(CLBR_ANY) .if \trace TRACE_IRQS_OFF .endif
@@ -862,7 +886,7 @@ KPROBE_ENTRY(error_entry) testl $3,CS(%rsp) je error_kernelspace error_swapgs: - swapgs + SWAPGS error_sti: movq %rdi,RDI(%rsp) CFI_REL_OFFSET rdi,RDI
@@ -874,7 +898,7 @@ error_sti: error_exit: movl %ebx,%eax RESTORE_REST - cli + DISABLE_INTERRUPTS(CLBR_NONE) TRACE_IRQS_OFF GET_THREAD_INFO(%rcx) testl %eax,%eax
@@ -911,12 +935,12 @@ ENTRY(load_gs_index) CFI_STARTPROC pushf CFI_ADJUST_CFA_OFFSET 8 - cli - swapgs + DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI)) + SWAPGS gs_change: movl %edi,%gs 2: mfence /* workaround */ - swapgs + SWAPGS popf CFI_ADJUST_CFA_OFFSET -8 ret
@@ -930,7 +954,7 @@ ENDPROC(load_gs_index) .section .fixup,"ax" /* running with kernelgs */ bad_gs: - swapgs /* switch back to user gs */ + SWAPGS /* switch back to user gs */ xorl %eax,%eax movl %eax,%gs jmp 2b
Signature · 1 line
1.4.4.2
PATCH 1/16 Wipe out traditional opt from x86_64 Makefile
In reply to Glauber de Oliveira CostaAmong other things, using -traditional as a gcc option stops us from using macro token pasting, which is a feature we heavily rely on.
There was still a use of -traditional in arch/x86/kernel/Makefile_64, which this patch removes.
I don't see any problems building kernels in my x86_64 box without -traditional.
--- arch/x86/kernel/Makefile_64 | 1 - 1 files changed, 0 insertions(+), 1 deletions(-)
arch/x86/kernel/Makefile_64 +0 -1
@@ -3,7 +3,6 @@ # extra-y := head_64.o head64.o init_task.o vmlinux.lds -EXTRA_AFLAGS := -traditional obj-y := process_64.o signal_64.o entry_64.o traps_64.o irq_64.o \ ptrace_64.o time_64.o ioport_64.o ldt_64.o setup_64.o i8259_64.o sys_x86_64.o \ x8664_ksyms_64.o i387_64.o syscall_64.o vsyscall_64.o \
Signature · 1 line
1.4.4.2