x86 paravirtualization infrastructure
Subsystems (4)
cover x86 paravirtualization infrastructure
The following patches introduce the core infrastructure needed to paravirtualize the 32-bit x86 Linux kernel. This is done by moving virtualization sensitive insn's or code paths to a function table, paravirt_ops. This structure can be populated with hypervisor specific calls or native stubs and currently support running on bare metal, VMI, Xen, or Lhype. These patches apply to 2.6.19-rc2-mm2 plus the last set of paravirt prep patches that Rusty sent.
thanks,
-chris
--
PATCH 1/7 header and stubs for paravirtualizing critical operations
Create a paravirt.h header for all the critical operations which need to be replaced with hypervisor calls, and include that instead of defining native operations, when CONFIG_PARAVIRT.
This patch does the dumbest possible replacement of paravirtualized
instructions: calls through a "paravirt_ops" structure. Currently these are function implementations of native hardware: hypervisors will override the ops structure with their own variants.
All the pv-ops functions are declared "fastcall" so that a specific register-based ABI is used, to make inlining assember easier.
--- arch/i386/Kconfig | 11 arch/i386/boot/compressed/misc.c | 1 arch/i386/kernel/Makefile | 1 arch/i386/kernel/asm-offsets.c | 10 arch/i386/kernel/entry.S | 34 +- arch/i386/kernel/i8259.c | 2 arch/i386/kernel/paravirt.c | 410 +++++++++++++++++++++++++++++ arch/i386/kernel/setup.c | 2 arch/i386/kernel/smpboot.c | 5 arch/i386/kernel/time.c | 15 - drivers/net/de600.c | 1 include/asm-i386/delay.h | 8 include/asm-i386/desc.h | 4 include/asm-i386/io.h | 10 include/asm-i386/irq.h | 10 include/asm-i386/irqflags.h | 42 +- include/asm-i386/mach-default/setup_arch.h | 2 include/asm-i386/msr.h | 5 include/asm-i386/paravirt.h | 291 ++++++++++++++++++++ include/asm-i386/processor.h | 15 - include/asm-i386/segment.h | 2 include/asm-i386/setup.h | 8 include/asm-i386/spinlock.h | 4 include/asm-i386/system.h | 16 - include/asm-i386/time.h | 41 ++ 25 files changed, 902 insertions(+), 48 deletions(-)
arch/i386/Kconfig +11 -0
@@ -197,6 +197,17 @@ config X86_ES7000 endchoice +config PARAVIRT + bool "Paravirtualization support (EXPERIMENTAL)" + depends on EXPERIMENTAL + help + Paravirtualization is a way of running multiple instances of + Linux on the same machine, under a hypervisor. This option + changes the kernel so it can modify itself when it is run + under a hypervisor, improving performance significantly. + However, when run without a hypervisor the kernel is + theoretically slower. If in doubt, say N. + config ACPI_SRAT bool default y
arch/i386/boot/compressed/misc.c +1 -0
@@ -9,6 +9,7 @@* High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996 */ +#undef CONFIG_PARAVIRT#include <linux/linkage.h> #include <linux/vmalloc.h> #include <linux/screen_info.h>
arch/i386/kernel/Makefile +1 -0
@@ -39,6 +39,7 @@ obj-$(CONFIG_VM86) += vm86.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_HPET_TIMER) += hpet.o obj-$(CONFIG_K8_NB) += k8.o +obj-$(CONFIG_PARAVIRT) += paravirt.o EXTRA_AFLAGS := -traditional
arch/i386/kernel/asm-offsets.c +10 -0
@@ -101,4 +101,14 @@ void foo(void)BLANK ();OFFSET (PDA_cpu, i386_pda, cpu_number);OFFSET (PDA_pcurrent, i386_pda, pcurrent); + +#ifdef CONFIG_PARAVIRT +BLANK (); +OFFSET (PARAVIRT_enabled, paravirt_ops, paravirt_enabled); +OFFSET (PARAVIRT_irq_disable, paravirt_ops, irq_disable); +OFFSET (PARAVIRT_irq_enable, paravirt_ops, irq_enable); +OFFSET (PARAVIRT_irq_enable_sysexit, paravirt_ops, irq_enable_sysexit); +OFFSET (PARAVIRT_iret, paravirt_ops, iret); +OFFSET (PARAVIRT_read_cr0, paravirt_ops, read_cr0); +#endif }
arch/i386/kernel/entry.S +27 -7
@@ -62,13 +62,6 @@ DF_MASK = 0x00000400 NT_MASK = 0x00004000 VM_MASK = 0x00020000 -/* These are replaces for paravirtualization */ -#define DISABLE_INTERRUPTS cli -#define ENABLE_INTERRUPTS sti -#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit -#define INTERRUPT_RETURN iret -#define GET_CR0_INTO_EAX movl %cr0, %eax - #ifdef CONFIG_PREEMPT #define preempt_stop DISABLE_INTERRUPTS; TRACE_IRQS_OFF #else
@@ -416,6 +409,20 @@ ldt_ss: jnz restore_nocheck testl $0x00400000, %eax # returning to 32bit stack? jnz restore_nocheck # allright, normal return + +#ifdef CONFIG_PARAVIRT + /* + * The kernel can't run on a non-flat stack if paravirt mode + * is active. Rather than try to fixup the high bits of + * ESP, bypass this code entirely. This may break DOSemu + * and/or Wine support in a paravirt VM, although the option + * is still available to implement the setting of the high + * 16-bits in the INTERRUPT_RETURN paravirt-op. + */ + cmpl $0, paravirt_ops+PARAVIRT_enabled + jne restore_nocheck +#endif + /* If returning to userspace with 16bit stack, * try to fix the higher word of ESP, as the CPU * won't restore it.
@@ -831,6 +838,19 @@ nmi_espfix_stack: .previous KPROBE_END(nmi) +#ifdef CONFIG_PARAVIRT +ENTRY(native_iret) +1: iret +.section __ex_table,"a" + .align 4 + .long 1b,iret_exc +.previous + +ENTRY(native_irq_enable_sysexit) + sti + sysexit +#endif + KPROBE_ENTRY(int3) RING0_INT_FRAME pushl $-1 # mark this as an int
arch/i386/kernel/i8259.c +1 -1
@@ -392,7 +392,7 @@ void __init init_ISA_irqs (void) } } -void __init init_IRQ (void ) +void __init native_init_IRQ (void ) {int i;
arch/i386/kernel/setup.c +1 -1
@@ -1404,7 +1404,7 @@ void __init setup_arch(char **cmdline_p)efi_init (); else {printk (KERN_INFO "BIOS-provided physical RAM map:\n" ); -print_memory_map (machine_specific_memory_setup ()); +print_memory_map (memory_setup ()); }copy_edd ();
arch/i386/kernel/smpboot.c +5 -0
@@ -33,6 +33,11 @@* Dave Jones : Report invalid combinations of Athlon CPUs. * Rusty Russell : Hacked into shape for new "hotplug" boot process. */ + +/* SMP boot always wants to use real time delay to allow sufficient time for +* the APs to come online */ +#define USE_REAL_TIME_DELAY +#include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h>
arch/i386/kernel/time.c +5 -10
@@ -56,6 +56,7 @@#include <asm/uaccess.h> #include <asm/processor.h> #include <asm/timer.h> +#include <asm/time.h> #include "mach_time.h"
@@ -116,10 +117,7 @@ static int set_rtc_mmss(unsigned long no/* gets recalled with irq locally disabled */ /* XXX - does irqsave resolve this? -johnstul */ spin_lock_irqsave (&rtc_lock, flags); -if (efi_enabled) - retval =efi_set_rtc_mmss (nowtime); -else - retval =mach_set_rtc_mmss (nowtime); + retval =set_wallclock (nowtime);spin_unlock_irqrestore (&rtc_lock, flags);return retval;
@@ -211,10 +209,7 @@ unsigned long read_persistent_clock(voidspin_lock_irqsave (&rtc_lock, flags); -if (efi_enabled) - retval =efi_get_time (); -else - retval =mach_get_cmos_time (); + retval =get_wallclock ();spin_unlock_irqrestore (&rtc_lock, flags);
@@ -280,7 +275,7 @@ static void __init hpet_time_init(void) printk("Using HPET for base-timer\n"); } -time_init_hook (); +do_time_init (); }#endif
@@ -296,5 +291,5 @@ void __init time_init(void) return; }#endif -time_init_hook (); +do_time_init (); }
drivers/net/de600.c +0 -1
@@ -43,7 +43,6 @@ static const char version[] = "de600.c:* modify the following "#define": (see <asm/io.h> for more info) #define REALLY_SLOW_IO */ -#define SLOW_IO_BY_JUMPING /* Looks "better" than dummy write to port 0x80 :-) */ /* use 0 for production, 1 for verification, >2 for debug */ #ifdef DE600_DEBUG
include/asm-i386/delay.h +8 -0
@@ -15,6 +15,13 @@ extern void __ndelay(unsigned long nsecsextern void __const_udelay (unsigned long usecs);extern void __delay (unsigned long loops); +#if defined(CONFIG_PARAVIRT ) && !defined(USE_REAL_TIME_DELAY ) +#define udelay (n) paravirt_ops.const_udelay((n) * 0x10c7ul) + +#define ndelay (n) paravirt_ops.const_udelay((n) * 5ul) + +#else /* !PARAVIRT || USE_REAL_TIME_DELAY */ +#define udelay (n) (__builtin_constant_p(n) ? \ ((n) > 20000 ? __bad_udelay() : __const_udelay((n) * 0x10c7ul)) : \ __udelay(n))
@@ -22,6 +29,7 @@ extern void __delay(unsigned long loops)#define ndelay (n) (__builtin_constant_p(n) ? \ ((n) > 20000 ? __bad_ndelay() : __const_udelay((n) * 5ul)) : \ __ndelay(n)) +#endif void use_tsc_delay (void );
include/asm-i386/desc.h +4 -0
@@ -55,6 +55,9 @@ static inline void pack_gate(u32 *low, u#define DESCTYPE_DPL3 0x60/* DPL-3 */ #define DESCTYPE_S 0x10/* !system */ +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #define load_TR_desc () __asm__ __volatile__("ltr %w0"::"q" (GDT_ENTRY_TSS*8))#define load_gdt (dtr) __asm__ __volatile("lgdt %0"::"m" (*dtr))
@@ -104,6 +107,7 @@ static inline void set_ldt(void *addr, u __asm__ __volatile__("lldt %w0"::"q" (GDT_ENTRY_LDT*8)); } } +#endif /* CONFIG_PARAVIRT */ static inline void _set_gate (int gate,unsigned int type,void *addr,unsigned short seg) {
include/asm-i386/io.h +6 -4
@@ -256,11 +256,11 @@ static inline void flush_write_buffers(v#endif /* __KERNEL__ */ -#ifdef SLOW_IO_BY_JUMPING -#define __SLOW_DOWN_IO "jmp 1f; 1: jmp 1f; 1:" -#else +#if defined(CONFIG_PARAVIRT ) +#include <asm/paravirt.h> +#else +#define __SLOW_DOWN_IO "outb %%al,$0x80;" -#endif static inline void slow_down_io (void ) { __asm__ __volatile__(
@@ -271,6 +271,8 @@ static inline void slow_down_io(void) { : : ); } +#endif +#ifdef CONFIG_X86_NUMAQ extern void *xquad_portio;/* Where the IO area was mapped */ #define XQUAD_PORT_ADDR (port, quad) (xquad_portio + (XQUAD_PORTIO_QUAD*quad) + port)
include/asm-i386/irq.h +10 -0
@@ -41,4 +41,14 @@ extern int irqbalance_disable(char *str)extern void fixup_irqs (cpumask_t map);#endif +void __init native_init_IRQ (void ); +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +static inline void init_IRQ (void ) +{ +native_init_IRQ (); +} +#endif /* CONFIG_PARAVIRT */ +#endif /* _ASM_IRQ_H */
include/asm-i386/irqflags.h +27 -15
69 lines not on this page · raw message
include/asm-i386/mach-default/setup_arch.h +2 -0
@@ -2,4 +2,6 @@/* no action for generic */ +#ifndef ARCH_SETUP #define ARCH_SETUP +#endif
include/asm-i386/msr.h +5 -0
@@ -1,6 +1,10 @@#ifndef __ASM_MSR_H#define __ASM_MSR_H +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +/* * Access to machine-specific registers (available on 586 and better only) * Note: the rd* operations modify the parameters directly (without using
@@ -77,6 +81,7 @@ static inline void wrmsrl (unsigned long __asm__ __volatile__("rdpmc" \ :"=a" (low),"=d" (high) \ : "c" (counter)) +#endif /* !CONFIG_PARAVIRT */ /* symbolic names for some interesting MSRs */ /* Intel defined MSRs. */
include/asm-i386/processor.h +12 -3
@@ -146,8 +146,8 @@ static inline void detect_ht(struct cpui#define X86_EFLAGS_VIP 0x00100000/* Virtual Interrupt Pending */ #define X86_EFLAGS_ID 0x00200000/* CPUID detection flag */ -static inline void __cpuid (unsigned int *eax,unsigned int *ebx, -unsigned int *ecx,unsigned int *edx) +static inline fastcall void native_cpuid(unsigned int *eax, unsigned int *ebx, +unsigned int *ecx,unsigned int *edx) {/* ecx is often an input as well as an output. */ __asm__("cpuid"
@@ -548,6 +548,12 @@ static inline void rep_nop(void)#define cpu_relax () rep_nop() +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define paravirt_enabled () 0 +#define __cpuid native_cpuid +static inline void load_esp0 (struct tss_struct *tss,struct thread_struct *thread) { tss->esp0 = thread->esp0;
@@ -570,10 +576,13 @@ static inline void load_esp0(struct tss_ : /* no output */ \ :"r" (value)) +#define set_iopl_mask native_set_iopl_mask +#endif /* CONFIG_PARAVIRT */ +/* * Set IOPL bits in EFLAGS from given mask */ -static inline void set_iopl_mask (unsigned mask) +static fastcall inline void native_set_iopl_mask(unsigned mask) {unsigned int reg; __asm__ __volatile__ ("pushfl;"
include/asm-i386/segment.h +2 -0
@@ -131,5 +131,7 @@#define SEGMENT_LDT 0x4#define SEGMENT_GDT 0x0 +#ifndef CONFIG_PARAVIRT #define get_kernel_rpl () 0#endif +#endif
include/asm-i386/setup.h +8 -0
@@ -70,6 +70,14 @@ extern unsigned char boot_params[PARAM_Sstruct e820entry ;char *__init machine_specific_memory_setup (void ); +#ifndef CONFIG_PARAVIRT +static inline char *memory_setup (void ) +{ +return machine_specific_memory_setup (); +} +#else +#include <asm/paravirt.h> +#endif int __init copy_e820_map (struct e820entry * biosmap,int nr_map);int __init sanitize_e820_map (struct e820entry * biosmap,char * pnr_map);
include/asm-i386/spinlock.h +4 -0
@@ -7,8 +7,12 @@#include <asm/processor.h> #include <linux/compiler.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #define CLI_STRING "cli"#define STI_STRING "sti" +#endif /* CONFIG_PARAVIRT */ /* * Your basic SMP spinlocks, allowing only a single CPU anywhere
include/asm-i386/system.h +10 -6
@@ -88,6 +88,9 @@ __asm__ __volatile__ ("movw %%dx,%1\n\t"#define savesegment (seg, value) \ asm volatile("mov %%" #seg ",%0":"=rm" (value)) +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #define read_cr0 () ({ \ unsigned int __dummy; \ __asm__ __volatile__( \
@@ -139,17 +142,18 @@ __asm__ __volatile__ ("movw %%dx,%1\n\t"#define write_cr4 (x) \ __asm__ __volatile__("movl %0,%%cr4": :"r" (x)) -/* -* Clear and set 'TS' bit respectively -*/ +#define wbinvd () \ + __asm__ __volatile__ ("wbinvd": : :"memory") + +/* Clear the 'TS' bit */ #define clts () __asm__ __volatile__ ("clts") +#endif /* CONFIG_PARAVIRT */ + +/* Set the 'TS' bit */ #define stts () write_cr0(8 | read_cr0())#endif /* __KERNEL__ */ -#define wbinvd () \ - __asm__ __volatile__ ("wbinvd": : :"memory") -static inline unsigned long get_limit (unsigned long segment) {unsigned long __limit;
arch/i386/kernel/paravirt.c new file +410 -0
410 lines not on this page · raw message
include/asm-i386/paravirt.h new file +291 -0
291 lines not on this page · raw message
include/asm-i386/time.h new file +41 -0
41 lines not on this page · raw message
--
Re: [PATCH 1/7] header and stubs for paravirtualizing critical operations
Can you add a high level comment to entry.S what paravirt mode is all about and perhaps a quick cheat sheet on the macros?
Quoted 3 lines
+/* SMP boot always wants to use real time delay to allow sufficient time for + * the APs to come online */
+#defineUSE_REAL_TIME_DELAY
That's ugly. Can't you call different wait functions for that case instead?
include/asm-i386/irq.h extern int irqbalance_disable(char *str) · line 52
Quoted 8 lines
include/asm-i386/irq.h +8
-0+#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +static inline void init_IRQ (void ) +{ +native_init_IRQ (); +} +#endif /* CONFIG_PARAVIRT */
You could probably avoid a lot of ifdefs by strategic use of __attribute__((weak))
include/asm-i386/desc.h static inline void pack_gate(u32 *low, u · line 60
Quoted 3 lines
include/asm-i386/desc.h +3
-0+#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else
This is probably a good candidate for rename to native + wrapper macros too. Otherwise we'll always have to hack two different places later.
-Andi
PATCH 2/7 Patch inline replacements for common paravirt operations.
In reply to Chris WrightIt turns out that the most called ops, by several orders of magnitude, are the interrupt manipulation ops. These are obvious candidates for patching, so mark them up and create infrastructure for it.
The method used is that the ops structure has a patch function, which is called for each place which needs to be patched: this returns a number of instructions (the rest are NOP-padded).
Usually we can spare a register (%eax) for the binary patched code to use, but in a couple of critical places in entry.S we can't: we make the clobbers explicit at the call site, and manually clobber the allowed registers in debug mode as an extra check.
--- arch/i386/kernel/alternative.c | 66 ++++++++++++-- arch/i386/kernel/entry.S | 26 ++--- arch/i386/kernel/module.c | 9 + arch/i386/kernel/paravirt.c | 44 +++++++++ arch/i386/kernel/vmlinux.lds.S | 6 + include/asm-i386/alternative.h | 3 include/asm-i386/irqflags.h | 4 include/asm-i386/paravirt.h | 185 +++++++++++++++++++++++++++++++++-------- include/asm-i386/spinlock.h | 5 - 9 files changed, 284 insertions(+), 64 deletions(-)
arch/i386/kernel/alternative.c +55 -11
@@ -123,6 +123,20 @@ static unsigned char** find_nop_table(vo#endif /* CONFIG_X86_64 */ +static void nop_out (void *insns,unsigned int len) +{ +unsigned char **noptable =find_nop_table (); + +while (len >0 ) { +unsigned int noplen = len; +if (noplen >ASM_NOP_MAX ) + noplen =ASM_NOP_MAX ; +memcpy (insns, noptable[noplen], noplen); + insns += noplen; + len -= noplen; + } +} +extern struct alt_instr __alt_instructions[], __alt_instructions_end[];extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];extern u8 *__smp_locks[], *__smp_locks_end[];
@@ -137,10 +151,9 @@ extern u8 __smp_alt_begin[], __smp_alt_evoid apply_alternatives (struct alt_instr *start,struct alt_instr *end) { -unsigned char **noptable =find_nop_table ();struct alt_instr *a;u8 *instr; -int diff, i, k; +int diff;DPRINTK ("%s: alt table %p -> %p\n" , __FUNCTION__, start, end);for (a = start; a < end; a++) {
@@ -158,13 +171,7 @@ void apply_alternatives(struct alt_instr#endif memcpy (instr, a->replacement, a->replacementlen); diff = a->instrlen - a->replacementlen; -/* Pad the rest with nops */ -for (i = a->replacementlen; diff >0 ; diff -= k, i += k) { - k = diff; -if (k >ASM_NOP_MAX ) - k =ASM_NOP_MAX ; -memcpy (a->instr + i, noptable[k], k); - } +nop_out (instr + a->replacementlen, diff); } }
@@ -208,7 +215,6 @@ static void alternatives_smp_lock(u8 **sstatic void alternatives_smp_unlock (u8 **start,u8 **end,u8 *text,u8 *text_end) { -unsigned char **noptable =find_nop_table ();u8 **ptr;for (ptr = start; ptr < end; ptr++) {
@@ -216,7 +222,7 @@ static void alternatives_smp_unlock(u8 *continue ;if (*ptr > text_end)continue ; - **ptr = noptable[1 ][0 ]; +nop_out (*ptr,1 ); }; }
@@ -342,6 +348,43 @@ void alternatives_smp_switch(int smp)#endif +#ifdef CONFIG_PARAVIRT +void apply_paravirt (struct paravirt_patch *start,struct paravirt_patch *end) +{ +struct paravirt_patch *p; +int i; + +for (p = start; p < end; p++) { +unsigned int used; + + used = paravirt_ops.patch (p->instrtype, p->clobbers, p->instr, + p->len); +#ifdef CONFIG_DEBUG_KERNEL +/* Deliberately clobber regs using "not %reg" to find bugs. */ +for (i =0 ; i <3 ; i++) { +if (p->len - used >=2 && (p->clobbers & (1 << i))) { +memcpy (p->instr + used,"\xf7\xd0" ,2 ); + p->instr[used+1 ] |= i; + used +=2 ; + } + } +#endif +/* Pad the rest with nops */ +nop_out (p->instr + used, p->len - used); + } + +/* Sync to be conservative, in case we patched following instructions */ +sync_core (); +} +extern struct paravirt_patch __start_parainstructions[], + __stop_parainstructions[]; +#else +void apply_paravirt (struct paravirt_patch *start,struct paravirt_patch *end) +{ +} +extern struct paravirt_patch *__start_parainstructions, *__stop_parainstructions; +#endif /* CONFIG_PARAVIRT */ +void __init alternative_instructions (void ) {unsigned long flags;
@@ -389,5 +432,6 @@ void __init alternative_instructions(voi alternatives_smp_switch(0); }#endif +apply_paravirt (__start_parainstructions, __stop_parainstructions);local_irq_restore (flags); }
arch/i386/kernel/entry.S +13 -13
@@ -63,9 +63,9 @@ NT_MASK = 0x00004000 VM_MASK = 0x00020000 #ifdef CONFIG_PREEMPT -#define preempt_stop DISABLE_INTERRUPTS; TRACE_IRQS_OFF +#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF #else -#define preempt_stop +#define preempt_stop(clobbers) #define resume_kernel restore_nocheck #endif
@@ -226,7 +226,7 @@ ENTRY(ret_from_fork) ALIGN RING0_PTREGS_FRAME ret_from_exception: - preempt_stop + preempt_stop(CLBR_ANY) ret_from_intr: GET_THREAD_INFO(%ebp) check_userspace:
@@ -237,7 +237,7 @@ check_userspace: jb resume_kernel # not returning to v8086 or userspace ENTRY(resume_userspace) - DISABLE_INTERRUPTS # make sure we don't miss an interrupt + DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret movl TI_flags(%ebp), %ecx
@@ -248,7 +248,7 @@ ENTRY(resume_userspace) #ifdef CONFIG_PREEMPT ENTRY(resume_kernel) - DISABLE_INTERRUPTS + DISABLE_INTERRUPTS(CLBR_ANY) cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? jnz restore_nocheck need_resched:
@@ -277,7 +277,7 @@ sysenter_past_esp: * No need to follow this irqs on/off section: the syscall * disabled irqs and here we enable it straight after entry: */ - ENABLE_INTERRUPTS + ENABLE_INTERRUPTS(CLBR_NONE) pushl $(__USER_DS) CFI_ADJUST_CFA_OFFSET 4 /*CFI_REL_OFFSET ss, 0*/
@@ -322,7 +322,7 @@ sysenter_past_esp: jae syscall_badsys call *sys_call_table(,%eax,4) movl %eax,PT_EAX(%esp) - DISABLE_INTERRUPTS + DISABLE_INTERRUPTS(CLBR_ECX|CLBR_EDX) TRACE_IRQS_OFF movl TI_flags(%ebp), %ecx testw $_TIF_ALLWORK_MASK, %cx
@@ -364,7 +364,7 @@ syscall_call: call *sys_call_table(,%eax,4) movl %eax,PT_EAX(%esp) # store the return value syscall_exit: - DISABLE_INTERRUPTS # make sure we don't miss an interrupt + DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret TRACE_IRQS_OFF
@@ -393,7 +393,7 @@ restore_nocheck_notrace: .section .fixup,"ax" iret_exc: TRACE_IRQS_ON - ENABLE_INTERRUPTS + ENABLE_INTERRUPTS(CLBR_NONE) pushl $0 # no error code pushl $do_iret_error jmp error_code
@@ -436,7 +436,7 @@ ldt_ss: CFI_ADJUST_CFA_OFFSET 4 pushl %eax CFI_ADJUST_CFA_OFFSET 4 - DISABLE_INTERRUPTS + DISABLE_INTERRUPTS(CLBR_EAX) TRACE_IRQS_OFF lss (%esp), %esp CFI_ADJUST_CFA_OFFSET -8
@@ -451,7 +451,7 @@ work_pending: jz work_notifysig work_resched: call schedule - DISABLE_INTERRUPTS # make sure we don't miss an interrupt + DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret TRACE_IRQS_OFF
@@ -507,7 +507,7 @@ syscall_exit_work: testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl jz work_pending TRACE_IRQS_ON - ENABLE_INTERRUPTS # could let do_syscall_trace() call + ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call # schedule() instead movl %esp, %eax movl $1, %edx
@@ -691,7 +691,7 @@ ENTRY(device_not_available) GET_CR0_INTO_EAX testl $0x4, %eax # EM (math emulation bit) jne device_not_available_emulate - preempt_stop + preempt_stop(CLBR_ANY) call math_state_restore jmp ret_from_exception device_not_available_emulate:
arch/i386/kernel/module.c +8 -1
@@ -109,7 +109,8 @@ int module_finalize(const Elf_Ehdr *hdr,const Elf_Shdr *sechdrs, struct module *me) { - const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL; + const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL, + *para =NULL ;char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -119,6 +120,8 @@ int module_finalize(const Elf_Ehdr *hdr, alt = s;if (!strcmp (".smp_locks" , secstrings + s->sh_name)) locks= s; +if (!strcmp (".parainstructions" , secstrings + s->sh_name)) + para = s; }if (alt) {
@@ -133,6 +136,10 @@ int module_finalize(const Elf_Ehdr *hdr, lseg, lseg + locks->sh_size, tseg, tseg + text->sh_size); } +if (para) { +void *pseg = (void *)para->sh_addr; +apply_paravirt (pseg, pseg + para->sh_size); + }return module_bug_finalize (hdr, sechdrs, me); }
arch/i386/kernel/paravirt.c +44 -0
56 lines not on this page · raw message
arch/i386/kernel/vmlinux.lds.S +6 -0
@@ -154,6 +154,12 @@ SECTIONS .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) { *(.altinstr_replacement) } + . = ALIGN(4); + __start_parainstructions = .; + .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) { + *(.parainstructions) + } + __stop_parainstructions = .; /* .exit.text is discard at runtime, not link time, to deal with references from .altinstructions and .eh_frame */ .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) { *(.exit.text) }
include/asm-i386/alternative.h +3 -0
@@ -118,4 +118,7 @@ static inline void alternatives_smp_swit#define LOCK_PREFIX ""#endif +struct paravirt_patch ; +void apply_paravirt (struct paravirt_patch *start,struct paravirt_patch *end); +#endif /* _I386_ALTERNATIVE_H */
include/asm-i386/irqflags.h +2 -2
@@ -79,8 +79,8 @@ static inline unsigned long __raw_local_ }#else -#define DISABLE_INTERRUPTS cli -#define ENABLE_INTERRUPTS sti +#define DISABLE_INTERRUPTS (clobbers) cli +#define ENABLE_INTERRUPTS (clobbers) sti#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit#define INTERRUPT_RETURN iret#define GET_CR0_INTO_EAX movl %cr0, %eax
include/asm-i386/paravirt.h +149 -36
213 lines not on this page · raw message
include/asm-i386/spinlock.h +4 -1
@@ -12,6 +12,7 @@#else #define CLI_STRING "cli"#define STI_STRING "sti" +#define CLI_STI_CLOBBERS #endif /* CONFIG_PARAVIRT */ /*
@@ -75,7 +76,9 @@ static inline void __raw_spin_lock_flags"jg 1b\n\t" "jmp 4b\n" "5:\n\t" - : "+m" (lock->slock) : "r" (flags) : "memory"); + : "+m" (lock->slock) + : "r" (flags) + : "memory" CLI_STI_CLOBBERS); } #endif
--
PATCH 3/7 More generic paravirtualization entry point.
In reply to Chris Wright1) Each hypervisor writes a probe function to detect whether we are running under that hypervisor. paravirt_probe() registers this function.
2) If vmlinux is booted with ring != 0, we call all the probe functions (with registers except %esp intact) in link order: the winner will not return.
--- arch/i386/kernel/Makefile | 2 ++ arch/i386/kernel/head.S | 33 +++++++++++++++++++++++++++++++++ arch/i386/kernel/paravirt.c | 6 +++++- arch/i386/kernel/vmlinux.lds.S | 6 ++++++ include/asm-i386/paravirt.h | 5 +++++ 5 files changed, 51 insertions(+), 1 deletion(-)
arch/i386/kernel/Makefile +2 -0
@@ -39,6 +39,8 @@ obj-$(CONFIG_VM86) += vm86.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_HPET_TIMER) += hpet.o obj-$(CONFIG_K8_NB) += k8.o + +# Make sure this is linked after any other paravirt_ops structs: see head.S obj-$(CONFIG_PARAVIRT) += paravirt.o EXTRA_AFLAGS := -traditional
arch/i386/kernel/head.S +33 -0
@@ -55,6 +55,12 @@ */ ENTRY(startup_32) +#ifdef CONFIG_PARAVIRT + movl %cs, %eax + testl $0x3, %eax + jnz startup_paravirt +#endif + /* * Set segments to known values. */
@@ -486,6 +492,33 @@ ignore_int: #endif iret +#ifdef CONFIG_PARAVIRT +startup_paravirt: + cld + movl $(init_thread_union+THREAD_SIZE),%esp + + /* We take pains to preserve all the regs. */ + pushl %edx + pushl %ecx + pushl %eax + + /* paravirt.o is last in link, and that probe fn never returns */ + pushl $__start_paravirtprobe +1: + movl 0(%esp), %eax + pushl (%eax) + movl 8(%esp), %eax + call *(%esp) + popl %eax + + movl 4(%esp), %eax + movl 8(%esp), %ecx + movl 12(%esp), %edx + + addl $4, (%esp) + jmp 1b +#endif + /* * Real beginning of normal "text" segment */
arch/i386/kernel/paravirt.c +5 -1
@@ -392,7 +392,11 @@ static int __init print_banner(void)return 0 ; }core_initcall (print_banner); - + +/* We simply declare start_kernel to be the paravirt probe of last resort. */ +asmlinkage void __init start_kernel(void); +paravirt_probe (start_kernel); +struct paravirt_ops paravirt_ops = { .name ="bare hardware" , .paravirt_enabled =0 ,
arch/i386/kernel/vmlinux.lds.S +6 -0
@@ -60,6 +60,12 @@ SECTIONS CONSTRUCTORS } :data + __start_paravirtprobe = .; + .paravirtprobe : AT(ADDR(.paravirtprobe) - LOAD_OFFSET) { + *(.paravirtprobe) + } + __stop_paravirtprobe = .; + . = ALIGN(4096); __nosave_begin = .; .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) { *(.data.nosave) }
include/asm-i386/paravirt.h +5 -0
@@ -120,6 +120,11 @@ struct paravirt_ops void (fastcall *iret)(void); }; +/* Mark a paravirt probe function. */ +#define paravirt_probe (fn) \ + static void (*__paravirtprobe_##fn)(void) __attribute_used__ \ + __attribute__((__section__(".paravirtprobe"))) = fn +extern struct paravirt_ops paravirt_ops;#define paravirt_enabled () (paravirt_ops.paravirt_enabled)
--
Re: [PATCH 3/7] More generic paravirtualization entry point.
Quoted 3 lines · Chris Wright
(quoted diff) +2
-0+ +/* We simply declare start_kernel to be the paravirt probe of last resort.*/ +asmlinkage void
__initstart_kernel(void);
Didn't rusty put that into a header recently?
-Andi
PATCH 4/7 Allow selected bug checks to be skipped by paravirt kernels
In reply to Chris WrightAllow selected bug checks to be skipped by paravirt kernels. The two most important are the F00F workaround (which is either done by the hypervisor, or not required), and the 'hlt' instruction check, which can break under some hypervisors.
--- arch/i386/kernel/cpu/intel.c | 2 +- include/asm-i386/bugs.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-)
arch/i386/kernel/cpu/intel.c +1 -1
@@ -107,7 +107,7 @@ static void __cpuinit init_intel(struct* Note that the workaround only should be initialized once... */ c->f00f_bug =0 ; -if ( c->x86 ==5 ) { +if (!paravirt_enabled () && c->x86 ==5 ) {static int f00f_workaround_enabled =0 ; c->f00f_bug =1 ;
include/asm-i386/bugs.h +4 -0
@@ -21,6 +21,7 @@#include <asm/processor.h> #include <asm/i387.h> #include <asm/msr.h> +#include <asm/paravirt.h> static int __init no_halt (char *s) {
@@ -91,6 +92,9 @@ static void __init check_fpu(void)static void __init check_hlt (void ) { +if (paravirt_enabled ()) +return ; +printk (KERN_INFO "Checking 'hlt' instruction... " );if (!boot_cpu_data.hlt_works_ok) {printk ("disabled\n" );
--
Re: [PATCH 4/7] Allow selected bug checks to be skipped by paravirt kernels
Quoted 4 lines · Chris Wright
Allow selected bug checks to be skipped by paravirt kernels. The two most important are the F00F workaround (which is either done by the hypervisor, or not required), and the 'hlt' instruction check, which can break under some hypervisors.
How can hlt check break? It is hlt;hlt;hlt, IIRC, that looks fairly innocent to me.
arch/i386/kernel/cpu/intel.c static void __cpuinit init_intel(struct · line 110
Quoted 8 lines
arch/i386/kernel/cpu/intel.c +1
-1@@ -107,7 +107,7 @@ static void __cpuinit init_intel(struct* Note that the workaround only should be initialized once... */ c->f00f_bug =0 ; -if ( c->x86 ==5 ) { +if (!paravirt_enabled () && c->x86 ==5 ) {
I'd do x86==5 check first... pentiums are not common any more.
Pavel
Signature · 2 lines
(english) http://www.livejournal.com/~pavelmachek (cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
Re: [PATCH 4/7] Allow selected bug checks to be skipped by paravirt kernels
Quoted 9 lines · Pavel Machek
arch/i386/kernel/cpu/intel.c static void __cpuinit init_intel(struct · line 110
Quoted 7 lines
+++
linux-2.6-pv/arch/i386/kernel/cpu/intel.carch/i386/kernel/cpu/intel.c +1
-1@@ -107,7 +107,7 @@ static void __cpuinit init_intel(struct* Note that the workaround only should be initialized once... */ c->f00f_bug =0 ; -if ( c->x86 ==5 ) { +if (!paravirt_enabled () && c->x86 ==5 ) {I'd do x86==5 check first... pentiums are not common any more.
It's not like paravirt_enabled will be common-case either, and is this isn't exactly a performance critical piece of code, it doesn't really matter which way around the checks are done.
Dave
Signature · 1 line
http://www.codemonkey.org.uk
Re: [PATCH 4/7] Allow selected bug checks to be skipped by paravirt kernels
In reply to Pavel MachekQuoted 11 lines · Pavel Machek
Quoted 5 lines · Chris Wright
Allow selected bug checks to be skipped by paravirt kernels. The two most important are the F00F workaround (which is either done by the hypervisor, or not required), and the 'hlt' instruction check, which can break under some hypervisors.
How can hlt check break? It is hlt;hlt;hlt, IIRC, that looks fairly innocent to me.
Not if you use tickless timers that don't generate interrupts to unhalt you, or if you delay ticks until the next scheduled timeout and you haven't yet scheduled any timeout. Both are likely in a hypervisor.
Zach
Re: [PATCH 4/7] Allow selected bug checks to be skipped by paravirt kernels
Hi!
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Allow selected bug checks to be skipped by paravirt kernels. The two most important are the F00F workaround (which is either done by the hypervisor, or not required), and the 'hlt' instruction check, which can break under some hypervisors.
Quoted 7 lines
Quoted 3 lines
How can hlt check break? It is hlt;hlt;hlt, IIRC, that looks fairly innocent to me.
Not if you use tickless timers that don't generate interrupts to unhalt you, or if you delay ticks until the next scheduled timeout and you haven't yet scheduled any timeout. Both are likely in a hypervisor.
Well.. but you are working around problem, instead of fixing it.
Tickless kernels are possible on normal machines, too.
Please fix it properly... probably by requesting timer 10msec in advance or something.
Pavel
Signature · 2 lines
(english) http://www.livejournal.com/~pavelmachek (cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
Re: [PATCH 4/7] Allow selected bug checks to be skipped by paravirt kernels
Quoted 7 lines · Pavel Machek
Quoted 6 lines
Quoted 4 lines
How can hlt check break? It is hlt;hlt;hlt, IIRC, that looks fairly innocent to me.
Not if you use tickless timers that don't generate interrupts to unhalt you, or if you delay ticks until the next scheduled timeout and you haven't yet scheduled any timeout. Both are likely in a hypervisor.
Quoted 8 lines
Well.. but you are working around problem, instead of fixing it.
Tickless kernels are possible on normal machines, too.
Please fix it properly... probably by requesting timer 10msec in advance or something.
Pavel
Well, I agree in spirit, but there is something to be said for keeping the code less complicated by removing these workarounds for broken processors. Preferably, we could remove the hlt check entirely, but then those people with these broken processors would not get the expected behavior of stalling during boot - that is the expected behavior of failure, correct? In any case, I added this workaround for the case when running under Xen. I would rather not add a dependence on timer scheduling to legacy bug checking code when the number of timer sources and tickless variations available is proliferating and the number of legacy processors that would even need this check is rapidly approaching zero.
Zach
PATCH 5/7 Allow disabling legacy power management modes with paravirt kernels
In reply to Chris WrightTwo legacy power management modes are much easier to just explicitly disable when running in paravirtualized mode - neither APM nor PnP is still relevant. The status of ACPI is still debatable, and noacpi is still a common enough boot parameter that it is not necessary to explicitly disable ACPI.
--- arch/i386/kernel/apm.c | 3 ++- drivers/pnp/pnpbios/core.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-)
arch/i386/kernel/apm.c +2 -1
@@ -231,6 +231,7 @@#include <asm/uaccess.h> #include <asm/desc.h> #include <asm/i8253.h> +#include <asm/paravirt.h> #include "io_ports.h"
@@ -2191,7 +2192,7 @@ static int __init apm_init(void)dmi_check_system (apm_dmi_table); -if (apm_info.bios.version ==0 ) { +if (apm_info.bios.version ==0 ||paravirt_enabled ()) {printk (KERN_INFO "apm: BIOS not found.\n" );return -ENODEV ; }
drivers/pnp/pnpbios/core.c +2 -1
@@ -530,7 +530,8 @@ static int __init pnpbios_init(void)if (check_legacy_ioport (PNPBIOS_BASE ))return -ENODEV ;#endif -if (pnpbios_disabled ||dmi_check_system (pnpbios_dmi_table)) { +if (pnpbios_disabled ||dmi_check_system (pnpbios_dmi_table) || +paravirt_enabled ()) {printk (KERN_INFO "PnPBIOS: Disabled\n" );return -ENODEV ; }
--
PATCH 6/7 Add APIC accessors to paravirt-ops.
In reply to Chris WrightAdd APIC accessors to paravirt-ops. Unfortunately, we need two write functions, as some older broken hardware requires workarounds for Pentium APIC errata - this is the purpose of apic_write_atomic.
--- arch/i386/kernel/paravirt.c | 28 ++++++++++++++++++++++++++++ include/asm-i386/apic.h | 5 ++++- include/asm-i386/paravirt.h | 27 +++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-)
arch/i386/kernel/paravirt.c +28 -0
@@ -28,6 +28,8 @@#include <asm/time.h> #include <asm/irq.h> #include <asm/delay.h> +#include <asm/fixmap.h> +#include <asm/apic.h> /* nop stub */ static void native_nop (void )
@@ -382,6 +384,26 @@ static fastcall void native_io_delay(voi asmvolatile ("outb %al,$0x80" ); } +#ifdef CONFIG_X86_LOCAL_APIC +/* +* Basic functions for reading and writing APIC registers +*/ +static fastcall void native_apic_write(unsigned long reg, unsigned long v) +{ + *((volatile unsigned long *)(APIC_BASE +reg)) = v; +} + +static fastcall void native_apic_write_atomic(unsigned long reg, unsigned long v) +{ +xchg ((volatile unsigned long *)(APIC_BASE +reg), v); +} + +static fastcall unsigned long native_apic_read (unsigned long reg) +{ +return *((volatile unsigned long *)(APIC_BASE +reg)); +} +#endif /* CONFIG_X86_LOCAL_APIC */ +/* These are in entry.S */ extern fastcall void native_iret(void); extern fastcall void native_irq_enable_sysexit(void);
@@ -452,6 +474,12 @@ struct paravirt_ops paravirt_ops = { .io_delay = native_io_delay, .const_udelay = __const_udelay, +#ifdef CONFIG_X86_LOCAL_APIC + .apic_write = native_apic_write, + .apic_write_atomic = native_apic_write_atomic, + .apic_read = native_apic_read, +#endif + .irq_enable_sysexit = native_irq_enable_sysexit, .iret = native_iret, };
include/asm-i386/apic.h +4 -1
@@ -37,7 +37,9 @@ extern void generic_apic_probe(void);/* * Basic functions accessing APICs. */ - +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else static __inlinevoid apic_write (unsigned long reg,unsigned long v) { *((volatile unsigned long *)(APIC_BASE +reg)) = v;
@@ -52,6 +54,7 @@ static __inline unsigned long apic_read( {return *((volatile unsigned long *)(APIC_BASE +reg)); } +#endif static __inline__void apic_wait_icr_idle (void ) {
include/asm-i386/paravirt.h +27 -0
@@ -115,6 +115,12 @@ struct paravirt_ops void (fastcall *io_delay)(void);void (*const_udelay)(unsigned long loops); +#ifdef CONFIG_X86_LOCAL_APIC + void (fastcall *apic_write)(unsigned long reg, unsigned long v); + void (fastcall *apic_write_atomic)(unsigned long reg, unsigned long v); + unsigned long (fastcall *apic_read)(unsigned long reg); +#endif +/* These two are jmp to, not actually called. */ void (fastcall *irq_enable_sysexit)(void); void (fastcall *iret)(void);
@@ -280,6 +286,27 @@ static inline void slow_down_io(void) {#endif } +#ifdef CONFIG_X86_LOCAL_APIC +/* +* Basic functions accessing APICs. +*/ +static __inlinevoid apic_write (unsigned long reg,unsigned long v) +{ + paravirt_ops.apic_write (reg,v); +} + +static __inlinevoid apic_write_atomic (unsigned long reg,unsigned long v) +{ + paravirt_ops.apic_write_atomic (reg,v); +} + +static __inlineunsigned long apic_read (unsigned long reg) +{ +return paravirt_ops.apic_read (reg); +} +#endif + +/* These all sit in the .parainstructions section to tell us what to patch. */ struct paravirt_patch {u8 *instr;/* original instructions */
--
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
arch/i386/kernel/paravirt.c static fastcall void native_io_delay(voi · line 387
Quoted 9 lines
/* nop stub */
static voidnative_nop(void)arch/i386/kernel/paravirt.c +1
-0@@ -382,6 +384,26 @@ static fastcall void native_io_delay(voi asmvolatile ("outb %al,$0x80" ); } +#ifdef CONFIG_X86_LOCAL_APIC
It would be nicer if you renamed the functions in apic.h to native_apic_* and then do
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#define apic_read native_apic_read
...
#endif
This way we wouldn't get that much duplication.
This might apply to at least some of the other paravirt ops too.
-Andi
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
Quoted 2 lines · Andi Kleen
It would be nicer if you renamed the functions in
apic.hto native_apic_* and then do
...
Quoted 1 line
This might apply to at least some of the other paravirt ops too.
Yes. I've done the obvious candidates below (as well as responding to some of your other points). Many ops are one-liners, and I don't want to cause too much additional churn.
Cheers!
Rusty.
Subject: Paravirtualization Kleenups
1) Add "cheatsheet" comments to entry.S about macros.
2) Use weak alias for init_IRQ -> native_init_IRQ in !CONFIG_PARAVIRT case. This removes an #ifdef.
3) Use shiny new start_kernel.h rather than another declaration.
4) Avoid duplication in paravirt.c: rename set_ldt to native_set_ldt, and use macro in !PARAVIRT case.
5) Same trick for apic ops.
There are other cases where we could use a renaming+macro similar trick to avoid duplication, but they're generally one-liners.
arch/i386/kernel/entry.S +13 -0
@@ -52,6 +52,19 @@ #include <asm/percpu.h> #include <asm/dwarf2.h> #include "irq_vectors.h" + +/* + * We use macros for low-level operations which need to be overridden + * for paravirtualization. The following will never clobber any registers: + * INTERRUPT_RETURN (aka. "iret") + * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") + * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). + * + * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must + * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). + * Allowing a register to be clobbered can shrink the paravirt replacement + * enough to patch inline, increasing performance. + */ #define nr_syscalls ((syscall_table_size)/4)
arch/i386/kernel/i8259.c +3 -0
@@ -392,6 +392,9 @@ void __init init_ISA_irqs (void) } } +/* Overridden in paravirt.c */ +void init_IRQ (void ) __attribute__((weak,alias ("native_init_IRQ" ))); +void __init native_init_IRQ (void ) {int i;
arch/i386/kernel/paravirt.c +6 -37
73 lines not on this page · raw message
include/asm-i386/apic.h +10 -4
@@ -40,21 +40,27 @@ extern void generic_apic_probe(void);#ifdef CONFIG_PARAVIRT #include <asm/paravirt.h> #else -static __inlinevoid apic_write (unsigned long reg,unsigned long v) +#define apic_write native_apic_write +#define apic_write_atomic native_apic_write_atomic +#define apic_read native_apic_read +#endif + +static __inline fastcall void native_apic_write(unsigned long reg, +unsigned long v) { *((volatile unsigned long *)(APIC_BASE +reg)) = v; } -static __inlinevoid apic_write_atomic (unsigned long reg,unsigned long v) +static __inline fastcall void native_apic_write_atomic(unsigned long reg, +unsigned long v) {xchg ((volatile unsigned long *)(APIC_BASE +reg), v); } -static __inlineunsigned long apic_read (unsigned long reg) +static __inlinefastcall unsigned long native_apic_read (unsigned long reg) {return *((volatile unsigned long *)(APIC_BASE +reg)); } -#endif static __inline__void apic_wait_icr_idle (void ) {
include/asm-i386/desc.h +5 -2
@@ -92,7 +92,11 @@ static inline void write_dt_entry(void * lp[1 ] = entry_high; } -static inline void set_ldt (void *addr,unsigned int entries) +#define set_ldt native_set_ldt +#endif /* CONFIG_PARAVIRT */ + +static inline fastcall void native_set_ldt(const void *addr, +unsigned int entries) {if (likely (entries ==0 )) __asm__ __volatile__("lldt %w0" ::"q" (0 ));
@@ -107,7 +111,6 @@ static inline void set_ldt(void *addr, u __asm__ __volatile__("lldt %w0" ::"q" (GDT_ENTRY_LDT *8 )); } } -#endif /* CONFIG_PARAVIRT */ static inline void _set_gate (int gate,unsigned int type,void *addr,unsigned short seg) {
include/asm-i386/irq.h +1 -8
@@ -41,14 +41,7 @@ extern void fixup_irqs(cpumask_t map);extern void fixup_irqs (cpumask_t map);#endif +void init_IRQ (void );void __init native_init_IRQ (void ); -#ifdef CONFIG_PARAVIRT -#include <asm/paravirt.h> -#else -static inline void init_IRQ (void ) -{ -native_init_IRQ (); -} -#endif /* CONFIG_PARAVIRT */ #endif /* _ASM_IRQ_H */
include/asm-i386/paravirt.h +0 -5
@@ -153,11 +153,6 @@ extern struct paravirt_ops paravirt_ops;extern struct paravirt_ops paravirt_ops;#define paravirt_enabled () (paravirt_ops.paravirt_enabled) - -static inline void init_IRQ (void ) -{ - paravirt_ops.init_IRQ (); -}static inline void load_esp0 (struct tss_struct *tss,struct thread_struct *thread)
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
Quoted 1 line
Subject: Paravirtualization Kleenups
Thanks.
Chris, can you please merge those into the original patchkit?
I could do it myself, but then retransmits from Chris would be difficult if anything else would need to be changed.
Also fixing that !-Os compile error in the original patches would be good.
-Andi
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
Quoted 1 line · * Andi Kleen ([email protected]) wrote:
Chris, can you please merge those into the original patchkit?
Sure, I'll fold those in.
Quoted 4 lines
I could do it myself, but then retransmits from Chris would be difficult if anything else would need to be changed.
Also fixing that !-Os compile error in the original patches would be good.
Hmm, builds fine here. If you have a .config and/or error message I'll fix it up.
thanks,
-chris
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
Quoted 7 lines · Chris Wright
Quoted 4 lines
I could do it myself, but then retransmits from Chris would be difficult if anything else would need to be changed.
Also fixing that !-Os compile error in the original patches would be good.
Hmm, builds fine here. If you have a .config and/or error message I'll fix it up.
I haven't tried it myself (my laptop was on battery all the time and I didn't want to drain it with a full rebuild ;-), there was just a report that it didn't work. Or maybe that was with an old patch. If it works it's fine.
-Andi
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
Quoted 3 lines · * Andi Kleen ([email protected]) wrote:
I haven't tried it myself (my laptop was on battery all the time and I didn't want to drain it with a full rebuild ;-), there was just a report that it didn't work. Or maybe that was with an old patch. If it works it's fine.
Ah yes, I see the report, (it's against a patch that has been redone), but I'll double check.
thanks,
-chris
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
In reply to Andi KleenQuoted 5 lines · Andi Kleen
Quoted 3 lines · Chris Wright
Quoted 3 lines
I could do it myself, but then retransmits from Chris would be difficult if anything else would need to be changed.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
Also fixing that !-Os compile error in the original patches would be good.
Hmm, builds fine here. If you have a .config and/or error message I'll fix it up.
I haven't tried it myself (my laptop was on battery all the time and I didn't want to drain it with a full rebuild ;-), there was just a report that it didn't work. Or maybe that was with an old patch. If it works it's fine.
The -Os thing was a red herring. It was a brokenpatch in the original 4 which for which I immediately sent a fixup to akpm. Here it is again below:
==
Move write_dt_entry back: moving it up breaks compile.
===================================================================
include/asm-i386/desc.h +11 -11
@@ -78,6 +78,17 @@ static inline void load_TLS(struct threa#undef C } +#define write_ldt_entry (dt, entry, low, high) write_dt_entry(dt,entry,low,high) +#define write_gdt_entry (dt, entry, low, high) write_dt_entry(dt,entry,low,high) +#define write_idt_entry (dt, entry, low, high) write_dt_entry(dt,entry,low,high) + +static inline void write_dt_entry (void *dt,int entry,u32 entry_low,u32 entry_high) +{ +u32 *lp = (u32 *)((char *)dt + entry*8 ); + lp[0 ] = entry_low; + lp[1 ] = entry_high; +} +static inline void set_ldt (void *addr,unsigned int entries) {if (likely (entries ==0 ))
@@ -92,17 +103,6 @@ static inline void set_ldt(void *addr, uwrite_gdt_entry (get_cpu_gdt_table (cpu),GDT_ENTRY_LDT , low, high); __asm__ __volatile__("lldt %w0" ::"q" (GDT_ENTRY_LDT *8 )); } -} - -#define write_ldt_entry (dt, entry, low, high) write_dt_entry(dt,entry,low,high) -#define write_gdt_entry (dt, entry, low, high) write_dt_entry(dt,entry,low,high) -#define write_idt_entry (dt, entry, low, high) write_dt_entry(dt,entry,low,high) - -static inline void write_dt_entry (void *dt,int entry,u32 entry_low,u32 entry_high) -{ -u32 *lp = (u32 *)((char *)dt + entry*8 ); - lp[0 ] = entry_low; - lp[1 ] = entry_high; }static inline void _set_gate (int gate,unsigned int type,void *addr,unsigned short seg)
Re: [PATCH 6/7] Add APIC accessors to paravirt-ops.
In reply to Andi KleenQuoted 10 lines · Andi Kleen
Quoted 1 line
Subject: Paravirtualization Kleenups
Thanks.
Chris, can you please merge those into the original patchkit?
I could do it myself, but then retransmits from Chris would be difficult if anything else would need to be changed.
Also fixing that !-Os compile error in the original patches would be good.
That is "prep-for-paravirt-desch-clearer-parameter-names-fix.patch" in rc4-mm1.
I'll follow with the updated series, although the cleanup patch was pretty clear by itself...
Rusty.
PATCH 1/7 paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Andi KleenCreate a paravirt.h header for all the critical operations which need to be replaced with hypervisor calls, and include that instead of defining native operations, when CONFIG_PARAVIRT.
This patch does the dumbest possible replacement of paravirtualized
instructions: calls through a "paravirt_ops" structure. Currently these are function implementations of native hardware: hypervisors will override the ops structure with their own variants.
All the pv-ops functions are declared "fastcall" so that a specific register-based ABI is used, to make inlining assember easier.
===================================================================
arch/i386/Kconfig +11 -0
@@ -196,6 +196,17 @@ config X86_ES7000 should say N here. endchoice + +config PARAVIRT + bool "Paravirtualization support (EXPERIMENTAL)" + depends on EXPERIMENTAL + help + Paravirtualization is a way of running multiple instances of + Linux on the same machine, under a hypervisor. This option + changes the kernel so it can modify itself when it is run + under a hypervisor, improving performance significantly. + However, when run without a hypervisor the kernel is + theoretically slower. If in doubt, say N. config ACPI_SRAT bool
===================================================================
arch/i386/boot/compressed/misc.c +1 -0
@@ -9,6 +9,7 @@* High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996 */ +#undef CONFIG_PARAVIRT#include <linux/linkage.h> #include <linux/vmalloc.h> #include <linux/screen_info.h>
===================================================================
arch/i386/kernel/Makefile +1 -0
@@ -39,6 +39,7 @@ obj-$(CONFIG_EARLY_PRINTK) += early_prin obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_HPET_TIMER) += hpet.o obj-$(CONFIG_K8_NB) += k8.o +obj-$(CONFIG_PARAVIRT) += paravirt.o EXTRA_AFLAGS := -traditional
===================================================================
arch/i386/kernel/asm-offsets.c +10 -0
@@ -101,4 +101,14 @@ void foo(void)BLANK ();OFFSET (PDA_cpu, i386_pda, cpu_number);OFFSET (PDA_pcurrent, i386_pda, pcurrent); + +#ifdef CONFIG_PARAVIRT +BLANK (); +OFFSET (PARAVIRT_enabled, paravirt_ops, paravirt_enabled); +OFFSET (PARAVIRT_irq_disable, paravirt_ops, irq_disable); +OFFSET (PARAVIRT_irq_enable, paravirt_ops, irq_enable); +OFFSET (PARAVIRT_irq_enable_sysexit, paravirt_ops, irq_enable_sysexit); +OFFSET (PARAVIRT_iret, paravirt_ops, iret); +OFFSET (PARAVIRT_read_cr0, paravirt_ops, read_cr0); +#endif }
===================================================================
arch/i386/kernel/entry.S +27 -7
@@ -61,13 +61,6 @@ DF_MASK = 0x00000400 DF_MASK = 0x00000400 NT_MASK = 0x00004000 VM_MASK = 0x00020000 - -/* These are replaces for paravirtualization */ -#define DISABLE_INTERRUPTS cli -#define ENABLE_INTERRUPTS sti -#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit -#define INTERRUPT_RETURN iret -#define GET_CR0_INTO_EAX movl %cr0, %eax #ifdef CONFIG_PREEMPT #define preempt_stop DISABLE_INTERRUPTS; TRACE_IRQS_OFF
@@ -416,6 +409,20 @@ ldt_ss: jnz restore_nocheck testl $0x00400000, %eax # returning to 32bit stack? jnz restore_nocheck # allright, normal return + +#ifdef CONFIG_PARAVIRT + /* + * The kernel can't run on a non-flat stack if paravirt mode + * is active. Rather than try to fixup the high bits of + * ESP, bypass this code entirely. This may break DOSemu + * and/or Wine support in a paravirt VM, although the option + * is still available to implement the setting of the high + * 16-bits in the INTERRUPT_RETURN paravirt-op. + */ + cmpl $0, paravirt_ops+PARAVIRT_enabled + jne restore_nocheck +#endif + /* If returning to userspace with 16bit stack, * try to fix the higher word of ESP, as the CPU * won't restore it.
@@ -830,6 +837,19 @@ 1: INTERRUPT_RETURN .long 1b,iret_exc .previous KPROBE_END(nmi) + +#ifdef CONFIG_PARAVIRT +ENTRY(native_iret) +1: iret +.section __ex_table,"a" + .align 4 + .long 1b,iret_exc +.previous + +ENTRY(native_irq_enable_sysexit) + sti + sysexit +#endif KPROBE_ENTRY(int3) RING0_INT_FRAME
===================================================================
arch/i386/kernel/i8259.c +4 -1
@@ -392,7 +392,10 @@ void __init init_ISA_irqs (void) } } -void __init init_IRQ (void ) +/* Overridden in paravirt.c */ +void init_IRQ (void ) __attribute__((weak,alias ("native_init_IRQ" ))); + +void __init native_init_IRQ (void ) {int i;
===================================================================
arch/i386/kernel/setup.c +1 -1
@@ -1405,7 +1405,7 @@ void __init setup_arch(char **cmdline_p)efi_init (); else {printk (KERN_INFO "BIOS-provided physical RAM map:\n" ); -print_memory_map (machine_specific_memory_setup ()); +print_memory_map (memory_setup ()); }copy_edd ();
===================================================================
arch/i386/kernel/smpboot.c +5 -0
@@ -33,6 +33,11 @@* Dave Jones : Report invalid combinations of Athlon CPUs. * Rusty Russell : Hacked into shape for new "hotplug" boot process. */ + +/* SMP boot always wants to use real time delay to allow sufficient time for +* the APs to come online */ +#define USE_REAL_TIME_DELAY +#include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h>
===================================================================
arch/i386/kernel/time.c +6 -11
@@ -56,6 +56,7 @@#include <asm/uaccess.h> #include <asm/processor.h> #include <asm/timer.h> +#include <asm/time.h> #include "mach_time.h"
@@ -116,10 +117,7 @@ static int set_rtc_mmss(unsigned long no/* gets recalled with irq locally disabled */ /* XXX - does irqsave resolve this? -johnstul */ spin_lock_irqsave (&rtc_lock, flags); -if (efi_enabled) - retval =efi_set_rtc_mmss (nowtime); -else - retval =mach_set_rtc_mmss (nowtime); + retval =set_wallclock (nowtime);spin_unlock_irqrestore (&rtc_lock, flags);return retval;
@@ -211,10 +209,7 @@ unsigned long read_persistent_clock(voidspin_lock_irqsave (&rtc_lock, flags); -if (efi_enabled) - retval =efi_get_time (); -else - retval =mach_get_cmos_time (); + retval =get_wallclock ();spin_unlock_irqrestore (&rtc_lock, flags);
@@ -280,7 +275,7 @@ static void __init hpet_time_init(void) printk("Using HPET for base-timer\n"); } -time_init_hook (); +do_time_init (); }#endif
@@ -296,5 +291,5 @@ void __init time_init(void) return; }#endif -time_init_hook (); -} +do_time_init (); +}
===================================================================
drivers/net/de600.c +0 -1
@@ -43,7 +43,6 @@ static const char version[] = "de600.c:* modify the following "#define": (see <asm/io.h> for more info) #define REALLY_SLOW_IO */ -#define SLOW_IO_BY_JUMPING /* Looks "better" than dummy write to port 0x80 :-) */ /* use 0 for production, 1 for verification, >2 for debug */ #ifdef DE600_DEBUG
===================================================================
include/asm-i386/delay.h +8 -0
@@ -15,6 +15,13 @@ extern void __const_udelay(unsigned longextern void __const_udelay (unsigned long usecs);extern void __delay (unsigned long loops); +#if defined(CONFIG_PARAVIRT ) && !defined(USE_REAL_TIME_DELAY ) +#define udelay (n) paravirt_ops.const_udelay((n) * 0x10c7ul) + +#define ndelay (n) paravirt_ops.const_udelay((n) * 5ul) + +#else /* !PARAVIRT || USE_REAL_TIME_DELAY */ +#define udelay (n) (__builtin_constant_p(n) ? \ ((n) > 20000 ? __bad_udelay() : __const_udelay((n) * 0x10c7ul)) : \ __udelay(n))
@@ -22,6 +29,7 @@ extern void __delay(unsigned long loops)#define ndelay (n) (__builtin_constant_p(n) ? \ ((n) > 20000 ? __bad_ndelay() : __const_udelay((n) * 5ul)) : \ __ndelay(n)) +#endif void use_tsc_delay (void );
===================================================================
include/asm-i386/desc.h +8 -1
@@ -55,6 +55,9 @@ static inline void pack_gate(u32 *low, u#define DESCTYPE_DPL3 0x60/* DPL-3 */ #define DESCTYPE_S 0x10/* !system */ +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #define load_TR_desc () __asm__ __volatile__("ltr %w0"::"q" (GDT_ENTRY_TSS*8))#define load_gdt (dtr) __asm__ __volatile("lgdt %0"::"m" (*dtr))
@@ -89,7 +92,11 @@ static inline void write_dt_entry(void * lp[1 ] = entry_high; } -static inline void set_ldt (void *addr,unsigned int entries) +#define set_ldt native_set_ldt +#endif /* CONFIG_PARAVIRT */ + +static inline fastcall void native_set_ldt(const void *addr, +unsigned int entries) {if (likely (entries ==0 )) __asm__ __volatile__("lldt %w0" ::"q" (0 ));
===================================================================
include/asm-i386/io.h +6 -4
@@ -256,11 +256,11 @@ static inline void flush_write_buffers(v#endif /* __KERNEL__ */ -#ifdef SLOW_IO_BY_JUMPING -#define __SLOW_DOWN_IO "jmp 1f; 1: jmp 1f; 1:" -#else +#if defined(CONFIG_PARAVIRT ) +#include <asm/paravirt.h> +#else +#define __SLOW_DOWN_IO "outb %%al,$0x80;" -#endif static inline void slow_down_io (void ) { __asm__ __volatile__(
@@ -270,6 +270,8 @@ static inline void slow_down_io(void) {#endif : : ); } + +#endif #ifdef CONFIG_X86_NUMAQ extern void *xquad_portio;/* Where the IO area was mapped */
===================================================================
include/asm-i386/irq.h +3 -0
@@ -41,4 +41,7 @@ extern void fixup_irqs(cpumask_t map);extern void fixup_irqs (cpumask_t map);#endif +void init_IRQ (void ); +void __init native_init_IRQ (void ); +#endif /* _ASM_IRQ_H */
===================================================================
include/asm-i386/irqflags.h +27 -15
69 lines not on this page · raw message
===================================================================
include/asm-i386/mach-default/setup_arch.h +2 -0
@@ -2,4 +2,6 @@/* no action for generic */ +#ifndef ARCH_SETUP #define ARCH_SETUP +#endif
===================================================================
include/asm-i386/msr.h +5 -0
@@ -1,5 +1,9 @@#ifndef __ASM_MSR_H#define __ASM_MSR_H + +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else /* * Access to machine-specific registers (available on 586 and better only)
@@ -77,6 +81,7 @@ static inline void wrmsrl (unsigned long __asm__ __volatile__("rdpmc" \ :"=a" (low),"=d" (high) \ : "c" (counter)) +#endif /* !CONFIG_PARAVIRT */ /* symbolic names for some interesting MSRs */ /* Intel defined MSRs. */
===================================================================
include/asm-i386/processor.h +12 -3
@@ -146,8 +146,8 @@ static inline void detect_ht(struct cpui#define X86_EFLAGS_VIP 0x00100000/* Virtual Interrupt Pending */ #define X86_EFLAGS_ID 0x00200000/* CPUID detection flag */ -static inline void __cpuid (unsigned int *eax,unsigned int *ebx, -unsigned int *ecx,unsigned int *edx) +static inline fastcall void native_cpuid(unsigned int *eax, unsigned int *ebx, +unsigned int *ecx,unsigned int *edx) {/* ecx is often an input as well as an output. */ __asm__("cpuid"
@@ -548,6 +548,12 @@ static inline void rep_nop(void)#define cpu_relax () rep_nop() +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define paravirt_enabled () 0 +#define __cpuid native_cpuid +static inline void load_esp0 (struct tss_struct *tss,struct thread_struct *thread) { tss->esp0 = thread->esp0;
@@ -570,10 +576,13 @@ static inline void load_esp0(struct tss_ : /* no output */ \ :"r" (value)) +#define set_iopl_mask native_set_iopl_mask +#endif /* CONFIG_PARAVIRT */ +/* * Set IOPL bits in EFLAGS from given mask */ -static inline void set_iopl_mask (unsigned mask) +static fastcall inline void native_set_iopl_mask(unsigned mask) {unsigned int reg; __asm__ __volatile__ ("pushfl;"
===================================================================
include/asm-i386/segment.h +2 -0
@@ -131,5 +131,7 @@#define SEGMENT_LDT 0x4#define SEGMENT_GDT 0x0 +#ifndef CONFIG_PARAVIRT #define get_kernel_rpl () 0#endif +#endif
===================================================================
include/asm-i386/setup.h +8 -0
@@ -70,6 +70,14 @@ struct e820entry;struct e820entry ;char *__init machine_specific_memory_setup (void ); +#ifndef CONFIG_PARAVIRT +static inline char *memory_setup (void ) +{ +return machine_specific_memory_setup (); +} +#else +#include <asm/paravirt.h> +#endif int __init copy_e820_map (struct e820entry * biosmap,int nr_map);int __init sanitize_e820_map (struct e820entry * biosmap,char * pnr_map);
===================================================================
include/asm-i386/spinlock.h +4 -0
@@ -7,8 +7,12 @@#include <asm/processor.h> #include <linux/compiler.h> +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #define CLI_STRING "cli"#define STI_STRING "sti" +#endif /* CONFIG_PARAVIRT */ /* * Your basic SMP spinlocks, allowing only a single CPU anywhere
===================================================================
include/asm-i386/system.h +12 -8
@@ -88,6 +88,9 @@ __asm__ __volatile__ ("movw %%dx,%1\n\t"#define savesegment (seg, value) \ asm volatile("mov %%" #seg ",%0":"=rm" (value)) +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else #define read_cr0 () ({ \ unsigned int __dummy; \ __asm__ __volatile__( \
@@ -139,16 +142,17 @@ __asm__ __volatile__ ("movw %%dx,%1\n\t"#define write_cr4 (x) \ __asm__ __volatile__("movl %0,%%cr4": :"r" (x)) -/* -* Clear and set 'TS' bit respectively -*/ -#define clts () __asm__ __volatile__ ("clts") -#define stts () write_cr0(8 | read_cr0()) - -#endif /* __KERNEL__ */ -#define wbinvd () \ __asm__ __volatile__ ("wbinvd": : :"memory") + +/* Clear the 'TS' bit */ +#define clts () __asm__ __volatile__ ("clts") +#endif /* CONFIG_PARAVIRT */ + +/* Set the 'TS' bit */ +#define stts () write_cr0(8 | read_cr0()) + +#endif /* __KERNEL__ */ static inline unsigned long get_limit (unsigned long segment) {
===================================================================
arch/i386/kernel/paravirt.c new file +399 -0
399 lines not on this page · raw message
===================================================================
include/asm-i386/paravirt.h new file +286 -0
286 lines not on this page · raw message
===================================================================
include/asm-i386/time.h new file +41 -0
41 lines not on this page · raw message
PATCH 2/7 paravirtualization: Patch inline replacements for common paravirt operations.
It turns out that the most called ops, by several orders of magnitude, are the interrupt manipulation ops. These are obvious candidates for patching, so mark them up and create infrastructure for it.
The method used is that the ops structure has a patch function, which is called for each place which needs to be patched: this returns a number of instructions (the rest are NOP-padded).
Usually we can spare a register (%eax) for the binary patched code to use, but in a couple of critical places in entry.S we can't: we make the clobbers explicit at the call site, and manually clobber the allowed registers in debug mode as an extra check.
===================================================================
arch/i386/kernel/alternative.c +55 -11
@@ -123,6 +123,20 @@ static unsigned char** find_nop_table(vo#endif /* CONFIG_X86_64 */ +static void nop_out (void *insns,unsigned int len) +{ +unsigned char **noptable =find_nop_table (); + +while (len >0 ) { +unsigned int noplen = len; +if (noplen >ASM_NOP_MAX ) + noplen =ASM_NOP_MAX ; +memcpy (insns, noptable[noplen], noplen); + insns += noplen; + len -= noplen; + } +} +extern struct alt_instr __alt_instructions[], __alt_instructions_end[];extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];extern u8 *__smp_locks[], *__smp_locks_end[];
@@ -137,10 +151,9 @@ extern u8 __smp_alt_begin[], __smp_alt_evoid apply_alternatives (struct alt_instr *start,struct alt_instr *end) { -unsigned char **noptable =find_nop_table ();struct alt_instr *a;u8 *instr; -int diff, i, k; +int diff;DPRINTK ("%s: alt table %p -> %p\n" , __FUNCTION__, start, end);for (a = start; a < end; a++) {
@@ -158,13 +171,7 @@ void apply_alternatives(struct alt_instr#endif memcpy (instr, a->replacement, a->replacementlen); diff = a->instrlen - a->replacementlen; -/* Pad the rest with nops */ -for (i = a->replacementlen; diff >0 ; diff -= k, i += k) { - k = diff; -if (k >ASM_NOP_MAX ) - k =ASM_NOP_MAX ; -memcpy (a->instr + i, noptable[k], k); - } +nop_out (instr + a->replacementlen, diff); } }
@@ -208,7 +215,6 @@ static void alternatives_smp_lock(u8 **sstatic void alternatives_smp_unlock (u8 **start,u8 **end,u8 *text,u8 *text_end) { -unsigned char **noptable =find_nop_table ();u8 **ptr;for (ptr = start; ptr < end; ptr++) {
@@ -216,7 +222,7 @@ static void alternatives_smp_unlock(u8 *continue ;if (*ptr > text_end)continue ; - **ptr = noptable[1 ][0 ]; +nop_out (*ptr,1 ); }; }
@@ -341,6 +347,43 @@ void alternatives_smp_switch(int smp) }#endif + +#ifdef CONFIG_PARAVIRT +void apply_paravirt (struct paravirt_patch *start,struct paravirt_patch *end) +{ +struct paravirt_patch *p; +int i; + +for (p = start; p < end; p++) { +unsigned int used; + + used = paravirt_ops.patch (p->instrtype, p->clobbers, p->instr, + p->len); +#ifdef CONFIG_DEBUG_KERNEL +/* Deliberately clobber regs using "not %reg" to find bugs. */ +for (i =0 ; i <3 ; i++) { +if (p->len - used >=2 && (p->clobbers & (1 << i))) { +memcpy (p->instr + used,"\xf7\xd0" ,2 ); + p->instr[used+1 ] |= i; + used +=2 ; + } + } +#endif +/* Pad the rest with nops */ +nop_out (p->instr + used, p->len - used); + } + +/* Sync to be conservative, in case we patched following instructions */ +sync_core (); +} +extern struct paravirt_patch __start_parainstructions[], + __stop_parainstructions[]; +#else +void apply_paravirt (struct paravirt_patch *start,struct paravirt_patch *end) +{ +} +extern struct paravirt_patch *__start_parainstructions, *__stop_parainstructions; +#endif /* CONFIG_PARAVIRT */ void __init alternative_instructions (void ) {
@@ -389,5 +432,6 @@ void __init alternative_instructions(voi alternatives_smp_switch(0); }#endif +apply_paravirt (__start_parainstructions, __stop_parainstructions);local_irq_restore (flags); }
===================================================================
arch/i386/kernel/entry.S +26 -13
@@ -53,6 +53,19 @@ #include <asm/dwarf2.h> #include "irq_vectors.h" +/* + * We use macros for low-level operations which need to be overridden + * for paravirtualization. The following will never clobber any registers: + * INTERRUPT_RETURN (aka. "iret") + * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") + * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). + * + * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must + * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). + * Allowing a register to be clobbered can shrink the paravirt replacement + * enough to patch inline, increasing performance. + */ + #define nr_syscalls ((syscall_table_size)/4) CF_MASK = 0x00000001
@@ -63,9 +76,9 @@ VM_MASK = 0x00020000 VM_MASK = 0x00020000 #ifdef CONFIG_PREEMPT -#define preempt_stop DISABLE_INTERRUPTS; TRACE_IRQS_OFF +#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF #else -#define preempt_stop +#define preempt_stop(clobbers) #define resume_kernel restore_nocheck #endif
@@ -226,7 +239,7 @@ ENTRY(ret_from_fork) ALIGN RING0_PTREGS_FRAME ret_from_exception: - preempt_stop + preempt_stop(CLBR_ANY) ret_from_intr: GET_THREAD_INFO(%ebp) check_userspace:
@@ -237,7 +250,7 @@ check_userspace: jb resume_kernel # not returning to v8086 or userspace ENTRY(resume_userspace) - DISABLE_INTERRUPTS # make sure we don't miss an interrupt + DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret movl TI_flags(%ebp), %ecx
@@ -248,7 +261,7 @@ ENTRY(resume_userspace) #ifdef CONFIG_PREEMPT ENTRY(resume_kernel) - DISABLE_INTERRUPTS + DISABLE_INTERRUPTS(CLBR_ANY) cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ? jnz restore_nocheck need_resched:
@@ -277,7 +290,7 @@ sysenter_past_esp: * No need to follow this irqs on/off section: the syscall * disabled irqs and here we enable it straight after entry: */ - ENABLE_INTERRUPTS + ENABLE_INTERRUPTS(CLBR_NONE) pushl $(__USER_DS) CFI_ADJUST_CFA_OFFSET 4 /*CFI_REL_OFFSET ss, 0*/
@@ -322,7 +335,7 @@ 1: movl (%ebp),%ebp jae syscall_badsys call *sys_call_table(,%eax,4) movl %eax,PT_EAX(%esp) - DISABLE_INTERRUPTS + DISABLE_INTERRUPTS(CLBR_ECX|CLBR_EDX) TRACE_IRQS_OFF movl TI_flags(%ebp), %ecx testw $_TIF_ALLWORK_MASK, %cx
@@ -364,7 +377,7 @@ syscall_call: call *sys_call_table(,%eax,4) movl %eax,PT_EAX(%esp) # store the return value syscall_exit: - DISABLE_INTERRUPTS # make sure we don't miss an interrupt + DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret TRACE_IRQS_OFF
@@ -393,7 +406,7 @@ 1: INTERRUPT_RETURN .section .fixup,"ax" iret_exc: TRACE_IRQS_ON - ENABLE_INTERRUPTS + ENABLE_INTERRUPTS(CLBR_NONE) pushl $0 # no error code pushl $do_iret_error jmp error_code
@@ -436,7 +449,7 @@ ldt_ss: CFI_ADJUST_CFA_OFFSET 4 pushl %eax CFI_ADJUST_CFA_OFFSET 4 - DISABLE_INTERRUPTS + DISABLE_INTERRUPTS(CLBR_EAX) TRACE_IRQS_OFF lss (%esp), %esp CFI_ADJUST_CFA_OFFSET -8
@@ -451,7 +464,7 @@ work_pending: jz work_notifysig work_resched: call schedule - DISABLE_INTERRUPTS # make sure we don't miss an interrupt + DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret TRACE_IRQS_OFF
@@ -507,7 +520,7 @@ syscall_exit_work: testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl jz work_pending TRACE_IRQS_ON - ENABLE_INTERRUPTS # could let do_syscall_trace() call + ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call # schedule() instead movl %esp, %eax movl $1, %edx
@@ -691,7 +704,7 @@ ENTRY(device_not_available) GET_CR0_INTO_EAX testl $0x4, %eax # EM (math emulation bit) jne device_not_available_emulate - preempt_stop + preempt_stop(CLBR_ANY) call math_state_restore jmp ret_from_exception device_not_available_emulate:
===================================================================
arch/i386/kernel/module.c +8 -1
@@ -109,7 +109,8 @@ int module_finalize(const Elf_Ehdr *hdr,const Elf_Shdr *sechdrs, struct module *me) { - const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL; + const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL, + *para =NULL ;char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
@@ -119,6 +120,8 @@ int module_finalize(const Elf_Ehdr *hdr, alt = s;if (!strcmp (".smp_locks" , secstrings + s->sh_name)) locks= s; +if (!strcmp (".parainstructions" , secstrings + s->sh_name)) + para = s; }if (alt) {
@@ -133,6 +136,10 @@ int module_finalize(const Elf_Ehdr *hdr, lseg, lseg + locks->sh_size, tseg, tseg + text->sh_size); } +if (para) { +void *pseg = (void *)para->sh_addr; +apply_paravirt (pseg, pseg + para->sh_size); + }return module_bug_finalize (hdr, sechdrs, me); }
===================================================================
arch/i386/kernel/paravirt.c +44 -0
56 lines not on this page · raw message
===================================================================
arch/i386/kernel/vmlinux.lds.S +6 -0
@@ -154,6 +154,12 @@ SECTIONS .altinstr_replacement : AT(ADDR(.altinstr_replacement) - LOAD_OFFSET) { *(.altinstr_replacement) } + . = ALIGN(4); + __start_parainstructions = .; + .parainstructions : AT(ADDR(.parainstructions) - LOAD_OFFSET) { + *(.parainstructions) + } + __stop_parainstructions = .; /* .exit.text is discard at runtime, not link time, to deal with references from .altinstructions and .eh_frame */ .exit.text : AT(ADDR(.exit.text) - LOAD_OFFSET) { *(.exit.text) }
===================================================================
include/asm-i386/alternative.h +3 -0
@@ -118,4 +118,7 @@ static inline void alternatives_smp_swit#define LOCK_PREFIX ""#endif +struct paravirt_patch ; +void apply_paravirt (struct paravirt_patch *start,struct paravirt_patch *end); +#endif /* _I386_ALTERNATIVE_H */
===================================================================
include/asm-i386/irqflags.h +2 -2
@@ -79,8 +79,8 @@ static inline unsigned long __raw_local_ }#else -#define DISABLE_INTERRUPTS cli -#define ENABLE_INTERRUPTS sti +#define DISABLE_INTERRUPTS (clobbers) cli +#define ENABLE_INTERRUPTS (clobbers) sti#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit#define INTERRUPT_RETURN iret#define GET_CR0_INTO_EAX movl %cr0, %eax
===================================================================
include/asm-i386/paravirt.h +150 -37
214 lines not on this page · raw message
===================================================================
include/asm-i386/spinlock.h +4 -1
@@ -12,6 +12,7 @@#else #define CLI_STRING "cli"#define STI_STRING "sti" +#define CLI_STI_CLOBBERS #endif /* CONFIG_PARAVIRT */ /*
@@ -75,7 +76,9 @@ static inline void __raw_spin_lock_flags"jg 1b\n\t" "jmp 4b\n" "5:\n\t" - : "+m" (lock->slock) : "r" (flags) : "memory"); + : "+m" (lock->slock) + : "r" (flags) + : "memory" CLI_STI_CLOBBERS); } #endif
PATCH 3/7 paravirtualization: More generic paravirtualization entry point.
1) Each hypervisor writes a probe function to detect whether we are running under that hypervisor. paravirt_probe() registers this function.
2) If vmlinux is booted with ring != 0, we call all the probe functions (with registers except %esp intact) in link order: the winner will not return.
===================================================================
arch/i386/kernel/Makefile +2 -0
@@ -39,6 +39,8 @@ obj-$(CONFIG_EARLY_PRINTK) += early_prin obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_HPET_TIMER) += hpet.o obj-$(CONFIG_K8_NB) += k8.o + +# Make sure this is linked after any other paravirt_ops structs: see head.S obj-$(CONFIG_PARAVIRT) += paravirt.o EXTRA_AFLAGS := -traditional
===================================================================
arch/i386/kernel/head.S +33 -0
@@ -54,6 +54,12 @@ * can. */ ENTRY(startup_32) + +#ifdef CONFIG_PARAVIRT + movl %cs, %eax + testl $0x3, %eax + jnz startup_paravirt +#endif /* * Set segments to known values.
@@ -486,6 +492,33 @@ ignore_int: #endif iret +#ifdef CONFIG_PARAVIRT +startup_paravirt: + cld + movl $(init_thread_union+THREAD_SIZE),%esp + + /* We take pains to preserve all the regs. */ + pushl %edx + pushl %ecx + pushl %eax + + /* paravirt.o is last in link, and that probe fn never returns */ + pushl $__start_paravirtprobe +1: + movl 0(%esp), %eax + pushl (%eax) + movl 8(%esp), %eax + call *(%esp) + popl %eax + + movl 4(%esp), %eax + movl 8(%esp), %ecx + movl 12(%esp), %edx + + addl $4, (%esp) + jmp 1b +#endif + /* * Real beginning of normal "text" segment */
===================================================================
arch/i386/kernel/paravirt.c +5 -1
@@ -19,6 +19,7 @@#include <linux/module.h> #include <linux/efi.h> #include <linux/bcd.h> +#include <linux/start_kernel.h> #include <asm/bug.h> #include <asm/paravirt.h>
@@ -381,7 +382,10 @@ static int __init print_banner(void)return 0 ; }core_initcall (print_banner); - + +/* We simply declare start_kernel to be the paravirt probe of last resort. */ +paravirt_probe (start_kernel); +struct paravirt_ops paravirt_ops = { .name ="bare hardware" , .paravirt_enabled =0 ,
===================================================================
arch/i386/kernel/vmlinux.lds.S +6 -0
@@ -60,6 +60,12 @@ SECTIONS CONSTRUCTORS } :data + __start_paravirtprobe = .; + .paravirtprobe : AT(ADDR(.paravirtprobe) - LOAD_OFFSET) { + *(.paravirtprobe) + } + __stop_paravirtprobe = .; + . = ALIGN(4096); __nosave_begin = .; .data_nosave : AT(ADDR(.data_nosave) - LOAD_OFFSET) { *(.data.nosave) }
===================================================================
include/asm-i386/paravirt.h +5 -0
@@ -119,6 +119,11 @@ struct paravirt_ops void (fastcall *irq_enable_sysexit)(void); void (fastcall *iret)(void); }; + +/* Mark a paravirt probe function. */ +#define paravirt_probe (fn) \ + static void (*__paravirtprobe_##fn)(void) __attribute_used__ \ + __attribute__((__section__(".paravirtprobe"))) = fnextern struct paravirt_ops paravirt_ops;
PATCH 4/7 paravirtualization: Allow selected bug checks to be skipped by paravirt kernels
Allow selected bug checks to be skipped by paravirt kernels. The two most important are the F00F workaround (which is either done by the hypervisor, or not required), and the 'hlt' instruction check, which can break under some hypervisors.
===================================================================
arch/i386/kernel/cpu/intel.c +1 -1
@@ -107,7 +107,7 @@ static void __cpuinit init_intel(struct* Note that the workaround only should be initialized once... */ c->f00f_bug =0 ; -if ( c->x86 ==5 ) { +if (!paravirt_enabled () && c->x86 ==5 ) {static int f00f_workaround_enabled =0 ; c->f00f_bug =1 ;
===================================================================
include/asm-i386/bugs.h +4 -0
@@ -21,6 +21,7 @@#include <asm/processor.h> #include <asm/i387.h> #include <asm/msr.h> +#include <asm/paravirt.h> static int __init no_halt (char *s) {
@@ -91,6 +92,9 @@ static void __init check_fpu(void)static void __init check_hlt (void ) { +if (paravirt_enabled ()) +return ; +printk (KERN_INFO "Checking 'hlt' instruction... " );if (!boot_cpu_data.hlt_works_ok) {printk ("disabled\n" );
PATCH 5/7 paravirtualization: Allow disabling legacy power management modes with paravirt kernels
Two legacy power management modes are much easier to just explicitly disable when running in paravirtualized mode - neither APM nor PnP is still relevant. The status of ACPI is still debatable, and noacpi is still a common enough boot parameter that it is not necessary to explicitly disable ACPI.
===================================================================
arch/i386/kernel/apm.c +2 -1
@@ -231,6 +231,7 @@#include <asm/uaccess.h> #include <asm/desc.h> #include <asm/i8253.h> +#include <asm/paravirt.h> #include "io_ports.h"
@@ -2191,7 +2192,7 @@ static int __init apm_init(void)dmi_check_system (apm_dmi_table); -if (apm_info.bios.version ==0 ) { +if (apm_info.bios.version ==0 ||paravirt_enabled ()) {printk (KERN_INFO "apm: BIOS not found.\n" );return -ENODEV ; }
===================================================================
drivers/pnp/pnpbios/core.c +2 -1
@@ -530,7 +530,8 @@ static int __init pnpbios_init(void)if (check_legacy_ioport (PNPBIOS_BASE ))return -ENODEV ;#endif -if (pnpbios_disabled ||dmi_check_system (pnpbios_dmi_table)) { +if (pnpbios_disabled ||dmi_check_system (pnpbios_dmi_table) || +paravirt_enabled ()) {printk (KERN_INFO "PnPBIOS: Disabled\n" );return -ENODEV ; }
PATCH 6/7 paravirtualization: Add APIC accessors to paravirt-ops.
Add APIC accessors to paravirt-ops. Unfortunately, we need two write functions, as some older broken hardware requires workarounds for Pentium APIC errata - this is the purpose of apic_write_atomic.
===================================================================
arch/i386/kernel/paravirt.c +8 -0
@@ -29,6 +29,8 @@#include <asm/time.h> #include <asm/irq.h> #include <asm/delay.h> +#include <asm/fixmap.h> +#include <asm/apic.h> /* nop stub */ static void native_nop (void )
@@ -441,6 +443,12 @@ struct paravirt_ops paravirt_ops = { .io_delay = native_io_delay, .const_udelay = __const_udelay, +#ifdef CONFIG_X86_LOCAL_APIC + .apic_write = native_apic_write, + .apic_write_atomic = native_apic_write_atomic, + .apic_read = native_apic_read, +#endif + .irq_enable_sysexit = native_irq_enable_sysexit, .iret = native_iret, };
===================================================================
include/asm-i386/apic.h +12 -3
@@ -37,18 +37,27 @@ extern void generic_apic_probe(void);/* * Basic functions accessing APICs. */ +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define apic_write native_apic_write +#define apic_write_atomic native_apic_write_atomic +#define apic_read native_apic_read +#endif -static __inlinevoid apic_write (unsigned long reg,unsigned long v) +static __inline fastcall void native_apic_write(unsigned long reg, +unsigned long v) { *((volatile unsigned long *)(APIC_BASE +reg)) = v; } -static __inlinevoid apic_write_atomic (unsigned long reg,unsigned long v) +static __inline fastcall void native_apic_write_atomic(unsigned long reg, +unsigned long v) {xchg ((volatile unsigned long *)(APIC_BASE +reg), v); } -static __inlineunsigned long apic_read (unsigned long reg) +static __inlinefastcall unsigned long native_apic_read (unsigned long reg) {return *((volatile unsigned long *)(APIC_BASE +reg)); }
===================================================================
include/asm-i386/paravirt.h +27 -0
@@ -114,6 +114,12 @@ struct paravirt_ops void (fastcall *io_delay)(void);void (*const_udelay)(unsigned long loops); + +#ifdef CONFIG_X86_LOCAL_APIC + void (fastcall *apic_write)(unsigned long reg, unsigned long v); + void (fastcall *apic_write_atomic)(unsigned long reg, unsigned long v); + unsigned long (fastcall *apic_read)(unsigned long reg); +#endif /* These two are jmp to, not actually called. */ void (fastcall *irq_enable_sysexit)(void);
@@ -275,6 +281,27 @@ static inline void slow_down_io(void) {#endif } +#ifdef CONFIG_X86_LOCAL_APIC +/* +* Basic functions accessing APICs. +*/ +static __inlinevoid apic_write (unsigned long reg,unsigned long v) +{ + paravirt_ops.apic_write (reg,v); +} + +static __inlinevoid apic_write_atomic (unsigned long reg,unsigned long v) +{ + paravirt_ops.apic_write_atomic (reg,v); +} + +static __inlineunsigned long apic_read (unsigned long reg) +{ +return paravirt_ops.apic_read (reg); +} +#endif + +/* These all sit in the .parainstructions section to tell us what to patch. */ struct paravirt_patch {u8 *instr;/* original instructions */
PATCH 7/7 paravirtualization: Add mmu virtualization to paravirt-ops.
Add the three bare TLB accessor functions to paravirt-ops. Most amusingly, flush_tlb is redefined on SMP, so I can't call the paravirt op flush_tlb. Instead, I chose to indicate the actual flush type, kernel (global) vs. user (non-global). Global in this sense means using the global bit in the page table entry, which makes TLB entries persistent across CR3 reloads, not global as in the SMP sense of invoking remote shootdowns, so the term is confusingly overloaded.
===================================================================
arch/i386/kernel/paravirt.c +109 -0
@@ -31,6 +31,7 @@#include <asm/delay.h> #include <asm/fixmap.h> #include <asm/apic.h> +#include <asm/tlbflush.h> /* nop stub */ static void native_nop (void )
@@ -373,6 +374,97 @@ static fastcall void native_io_delay(voi { asmvolatile ("outb %al,$0x80" ); } + +static fastcall void native_flush_tlb(void) +{ +__native_flush_tlb (); +} + +/* +* Global pages have to be flushed a bit differently. Not a real +* performance problem because this does not happen often. +*/ +static fastcall void native_flush_tlb_global(void) +{ +__native_flush_tlb_global (); +} + +static fastcall void native_flush_tlb_single(u32 addr) +{ +__native_flush_tlb_single (addr); +} + +#ifndef CONFIG_X86_PAE +static fastcall void native_set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; +} + +static fastcall void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; +} + +static fastcall void native_set_pmd(pmd_t *pmdp, pmd_t pmdval) +{ + *pmdp = pmdval; +} + +#else /* CONFIG_X86_PAE */ + +static fastcall void native_set_pte(pte_t *ptep, pte_t pte) +{ + ptep->pte_high = pte.pte_high; +smp_wmb (); + ptep->pte_low = pte.pte_low; +} + +static fastcall void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pte) +{ + ptep->pte_high = pte.pte_high; +smp_wmb (); + ptep->pte_low = pte.pte_low; +} + +static fastcall void native_set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) +{ + ptep->pte_low =0 ; +smp_wmb (); + ptep->pte_high = pte.pte_high; +smp_wmb (); + ptep->pte_low = pte.pte_low; +} + +static fastcall void native_set_pte_atomic(pte_t *ptep, pte_t pteval) +{ +set_64bit ((unsigned long long *)ptep,pte_val (pteval)); +} + +static fastcall void native_set_pmd(pmd_t *pmdp, pmd_t pmdval) +{ +set_64bit ((unsigned long long *)pmdp,pmd_val (pmdval)); +} + +static fastcall void native_set_pud(pud_t *pudp, pud_t pudval) +{ + *pudp = pudval; +} + +static fastcall void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + ptep->pte_low =0 ; +smp_wmb (); + ptep->pte_high =0 ; +} + +static fastcall void native_pmd_clear(pmd_t *pmd) +{ +u32 *tmp = (u32 *)pmd; + *tmp =0 ; +smp_wmb (); + *(tmp +1 ) =0 ; +} +#endif /* CONFIG_X86_PAE */ /* These are in entry.S */ extern fastcall void native_iret(void);
@@ -449,6 +541,23 @@ struct paravirt_ops paravirt_ops = { .apic_read = native_apic_read, #endif + .flush_tlb_user = native_flush_tlb, + .flush_tlb_kernel = native_flush_tlb_global, + .flush_tlb_single = native_flush_tlb_single, + + .set_pte = native_set_pte, + .set_pte_at = native_set_pte_at, + .set_pmd = native_set_pmd, + .pte_update = (void *)native_nop, + .pte_update_defer = (void *)native_nop, +#ifdef CONFIG_X86_PAE + .set_pte_atomic = native_set_pte_atomic, + .set_pte_present = native_set_pte_present, + .set_pud = native_set_pud, + .pte_clear = native_pte_clear, + .pmd_clear = native_pmd_clear, +#endif + .irq_enable_sysexit = native_irq_enable_sysexit, .iret = native_iret, };
===================================================================
arch/i386/mm/boot_ioremap.c +1 -0
@@ -16,6 +16,7 @@*/ #undef CONFIG_X86_PAE +#undef CONFIG_PARAVIRT#include <asm/page.h> #include <asm/pgtable.h> #include <asm/tlbflush.h>
===================================================================
include/asm-i386/paravirt.h +75 -0
99 lines not on this page · raw message
===================================================================
include/asm-i386/pgtable-2level.h +4 -1
@@ -11,11 +11,14 @@* within a page table are directly modified. Thus, the following * hook is made available. */ +#ifndef CONFIG_PARAVIRT #define set_pte (pteptr, pteval) (*(pteptr) = pteval)#define set_pte_at (mm,addr,ptep,pteval) set_pte(ptep,pteval) +#define set_pmd (pmdptr, pmdval) (*(pmdptr) = (pmdval)) +#endif +#define set_pte_atomic (pteptr, pteval) set_pte(pteptr,pteval)#define set_pte_present (mm,addr,ptep,pteval) set_pte_at(mm,addr,ptep,pteval) -#define set_pmd (pmdptr, pmdval) (*(pmdptr) = (pmdval))#define pte_clear (mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)#define pmd_clear (xp) do { set_pmd(xp, __pmd(0)); } while (0)
===================================================================
include/asm-i386/pgtable-3level.h +28 -26
72 lines not on this page · raw message
===================================================================
include/asm-i386/pgtable.h +3 -1
@@ -15,6 +15,7 @@#include <asm/processor.h> #include <asm/fixmap.h> #include <linux/threads.h> +#include <asm/paravirt.h> #ifndef _I386_BITOPS_H#include <asm/bitops.h>
@@ -246,6 +247,7 @@ static inline pte_t pte_mkhuge(pte_t pte# include <asm/pgtable-2level.h> #endif +#ifndef CONFIG_PARAVIRT /* * Rules for using pte_update - it must be called after any PTE update which * has not been done using the set_pte / clear_pte interfaces. It is used by
@@ -261,7 +263,7 @@ static inline pte_t pte_mkhuge(pte_t pte*/ #define pte_update (mm, addr, ptep) do { } while (0)#define pte_update_defer (mm, addr, ptep) do { } while (0) - +#endif /* * We only update the dirty/accessed state if we set
===================================================================
include/asm-i386/tlbflush.h +13 -5
@@ -4,7 +4,15 @@#include <linux/mm.h> #include <asm/processor.h> -#define __flush_tlb () \ +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define __flush_tlb () __native_flush_tlb() +#define __flush_tlb_global () __native_flush_tlb_global() +#define __flush_tlb_single (addr) __native_flush_tlb_single(addr) +#endif + +#define __native_flush_tlb () \ do { \ unsigned int tmpreg; \ \
@@ -19,7 +27,7 @@* Global pages have to be flushed a bit differently. Not a real * performance problem because this does not happen often. */ -#define __flush_tlb_global () \ +#define __native_flush_tlb_global () \ do { \ unsigned int tmpreg, cr4, cr4_orig; \ \
@@ -36,6 +44,9 @@ : "memory"); \ } while (0) +#define __native_flush_tlb_single (addr) \ + __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory") +# define __flush_tlb_all () \ do { \ if (cpu_has_pge) \
@@ -45,9 +56,6 @@ } while (0)#define cpu_has_invlpg (boot_cpu_data.x86 > 3) - -#define __flush_tlb_single (addr) \ - __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory")#ifdef CONFIG_X86_INVLPG # define __flush_tlb_one (addr) __flush_tlb_single(addr)
Re: [PATCH 6/7] paravirtualization: Add APIC accessors to paravirt-ops.
In reply to Rusty Russellinclude/asm-i386/paravirt.h static inline void slow_down_io(void) { · line 298
Quoted 3 lines · On Wed, 01 Nov 2006 21:32:30 +1100 Rusty Russell <[email protected]> wrote:
include/asm-i386/paravirt.h +3
-0+static __inline void apic_write(unsigned long reg, unsigned long v) +static __inline void apic_write_atomic(unsigned long reg, unsigned long v) +static __inline unsigned long apic_read(unsigned long reg)
Just `inline', please.
Re: [PATCH 6/7] paravirtualization: Add APIC accessors to paravirt-ops.
Quoted 8 lines · Andrew Morton
Review
Quoted 3 lines · On Wed, 01 Nov 2006 21:32:30 +1100 Rusty Russell <[email protected]> wrote:
(quoted diff) +3
-0+static __inline void apic_write(unsigned long reg, unsigned long v) +static __inline void apic_write_atomic(unsigned long reg, unsigned long v) +static __inline unsigned long apic_read(unsigned long reg)Just `inline', please.
akpm says: "Just `inline', please."
include/asm-i386/paravirt.h +3 -3
@@ -304,17 +304,17 @@ static inline void slow_down_io(void) {/* * Basic functions accessing APICs. */ -static __inlinevoid apic_write (unsigned long reg,unsigned long v) +static inline void apic_write (unsigned long reg,unsigned long v) { paravirt_ops.apic_write (reg,v); } -static __inlinevoid apic_write_atomic (unsigned long reg,unsigned long v) +static inline void apic_write_atomic (unsigned long reg,unsigned long v) { paravirt_ops.apic_write_atomic (reg,v); } -static __inlineunsigned long apic_read (unsigned long reg) +static inline unsigned long apic_read (unsigned long reg) {return paravirt_ops.apic_read (reg); }
Re: [PATCH 4/7] paravirtualization: Allow selected bug checks to be skipped by paravirt kernels
In reply to Rusty Russellinclude/asm-i386/bugs.h line 24
Quoted 7 lines · On Wed, 01 Nov 2006 21:30:43 +1100 Rusty Russell <[email protected]> wrote:
include/asm-i386/bugs.h +1
-0@@ -21,6 +21,7 @@#include <asm/processor.h> #include <asm/i387.h> #include <asm/msr.h> +#include <asm/paravirt.h>
In many other places you have
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
...
But not here.
Making <asm/paravirt.h> invulnerable would be the more typical approach.
Re: [PATCH 4/7] paravirtualization: Allow selected bug checks to be skipped by paravirt kernels
Quoted 15 lines · Andrew Morton
include/asm-i386/bugs.h line 25
Quoted 8 lines · On Wed, 01 Nov 2006 21:30:43 +1100 Rusty Russell <[email protected]> wrote:
include/asm-i386/bugs.h +1
-0@@ -21,6 +21,7 @@#include <asm/processor.h> #include <asm/i387.h> #include <asm/msr.h> +#include <asm/paravirt.h> In many other places you have
Quoted 7 lines
#ifdef
CONFIG_PARAVIRT
#include <asm/paravirt.h>
...But not here.
Making <
asm/paravirt.h> invulnerable would be the more typical approach.
CONFIG_PARAVIRT is not being used to guard asm/paravirt.h from multiple inclusion. In places where it is being used to guard #include <asm/paravirt.h>, the idea is that asm/paravirt.h defines various inlines/macros which would otherwise be defined in the header. So, for example, asm/desc.h would normally define load_gdt() in the !CONFIG_PARAVIRT case, but asm/paravirt.h defines it when CONFIG_PARAVIRT is enabled.
In this case, asm/paravirt.h included because we need the definition for paravirt_enabled(), not because it is replacing any of bugs.h's definitions.
J
Re: [PATCH 4/7] paravirtualization: Allow selected bug checks to be skipped by paravirt kernels
In reply to Andrew MortonQuoted 13 lines · Andrew Morton
include/asm-i386/bugs.h line 24
Quoted 7 lines · On Wed, 01 Nov 2006 21:30:43 +1100 Rusty Russell <[email protected]> wrote:
include/asm-i386/bugs.h +1
-0@@ -21,6 +21,7 @@#include <asm/processor.h> #include <asm/i387.h> #include <asm/msr.h> +#include <asm/paravirt.h> In many other places you have
Quoted 7 lines
#ifdef
CONFIG_PARAVIRT
#include <asm/paravirt.h>
...But not here.
Making <
asm/paravirt.h> invulnerable would be the more typical approach.
It *is* actually safe. The "#ifdef CONFIG_PARAVIRT / #include <asm/paravirt.h> / #else / <... native versions...>" is to give a big hint to the reader to look in paravirt.h for the real definitions.
Originally I had a noparavirt.h where all these lived, and people hated it. So we did it this way, which minimizes churn.
Rusty.
Re: [PATCH 2/7] paravirtualization: Patch inline replacements for common paravirt operations.
In reply to Rusty Russellarch/i386/kernel/alternative.c void alternatives_smp_switch(int smp) · line 363
Quoted 12 lines · On Wed, 01 Nov 2006 21:28:13 +1100 Rusty Russell <[email protected]> wrote:
arch/i386/kernel/alternative.c +12
-0+void apply_paravirt (struct paravirt_patch *start,struct paravirt_patch *end) +{ +struct paravirt_patch *p; +int i; + +for (p = start; p < end; p++) { +unsigned int used; + + used = paravirt_ops.patch (p->instrtype, p->clobbers, p->instr, + p->len); +#ifdef CONFIG_DEBUG_KERNEL +/* Deliberately clobber regs using "not %reg" to find bugs. */
That would be considered to be abusive of CONFIG_DEBUG_KERNEL. A CONFIG_DEBUG_PARAVIRT which depends on CONFIG_DEBUG_KERNEL would be more harmonious.
Re: [PATCH 2/7] paravirtualization: Patch inline replacements for common paravirt operations.
Quoted 8 lines · Andrew Morton
Review
Quoted 2 lines · On Wed, 01 Nov 2006 21:28:13 +1100 Rusty Russell <[email protected]> wrote:
(quoted diff) +2
-0+#ifdef CONFIG_DEBUG_KERNEL + /* Deliberately clobber regs using "not %reg" to find bugs. */That would be considered to be abusive of
CONFIG_DEBUG_KERNEL. ACONFIG_DEBUG_PARAVIRTwhich depends onCONFIG_DEBUG_KERNELwould be more harmonious.
I wasn't sure. Making a config option for what is a one-liner seemed overkill.
==
Don't abuse CONFIG_DEBUG_KERNEL, add CONFIG_DEBUG_PARAVIRT.
arch/i386/Kconfig.debug +10 -0
@@ -87,4 +87,14 @@ config DOUBLEFAULT option saves about 4k and might cause you much additional grey hair. +config DEBUG_PARAVIRT + bool "Enable some paravirtualization debugging" + default y + depends on PARAVIRT && DEBUG_KERNEL + help + Currently deliberately clobbers regs which are allowed to be + clobbered in inlined paravirt hooks, even in native mode. + If turning this off solves a problem, then DISABLE_INTERRUPTS() or + ENABLE_INTERRUPTS() is lying about what registers can be clobbered. + endmenu
arch/i386/kernel/alternative.c +1 -1
@@ -359,7 +359,7 @@ void apply_paravirt(struct paravirt_patc used = paravirt_ops.patch (p->instrtype, p->clobbers, p->instr, p->len); -#ifdef CONFIG_DEBUG_KERNEL +#ifdef CONFIG_DEBUG_PARAVIRT /* Deliberately clobber regs using "not %reg" to find bugs. */ for (i =0 ; i <3 ; i++) {if (p->len - used >=2 && (p->clobbers & (1 << i))) {
Re: [PATCH 2/7] paravirtualization: Patch inline replacements for common paravirt operations.
Quoted 13 lines · Rusty Russell
Quoted 10 lines · Andrew Morton
Review
Quoted 3 lines · On Wed, 01 Nov 2006 21:28:13 +1100 Rusty Russell <[email protected]> wrote:
(quoted diff) +2
-0+#ifdef CONFIG_DEBUG_KERNEL + /* Deliberately clobber regs using "not %reg" to find bugs. */That would be considered to be abusive of
CONFIG_DEBUG_KERNEL. ACONFIG_DEBUG_PARAVIRTwhich depends onCONFIG_DEBUG_KERNELwould be more harmonious.
Quoted 3 lines
I wasn't sure. Making a config option for what is a one-liner seemed overkill.
I have further stuff in my vmi-debug patch that can use CONFIG_DEBUG_PARAVIRT as well :)
Zach
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Rusty RussellQuoted 11 lines · Rusty Russell
Create a
paravirt.hheader for all the critical operations which need to be replaced with hypervisor calls, and include that instead of defining native operations, whenCONFIG_PARAVIRT.This patch does the dumbest possible replacement of paravirtualized
instructions: calls through a "paravirt_ops" structure. Currently these are function implementations of native hardware: hypervisors will override the ops structure with their own variants.All the pv-ops functions are declared "fastcall" so that a specific register-based ABI is used, to make inlining assember easier.
this is a lot of infrastructure... do we have more than 1 user of this yet that wants to get merged in mainline?
Signature · 2 lines
if you want to mail me at work (you don't), use arjan (at) linux.intel.com Test the interaction between Linux and your BIOS via http://www.linuxfirmwarekit.org
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 2 lines · Arjan van de Ven
this is a lot of infrastructure... do we have more than 1 user of this yet that wants to get merged in mainline?
AFAIK xen, vmi, lhype (and native ops).
-Andi
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Arjan van de VenQuoted 6 lines · Arjan van de Ven
Quoted 3 lines · Rusty Russell
Create a
paravirt.hheader for all the critical operations which need to be replaced with hypervisor calls, and include that instead of defining native operations, whenCONFIG_PARAVIRT.this is a lot of infrastructure... do we have more than 1 user of this yet that wants to get merged in mainline?
Yep. Xen and VMI both have patches on top of this pending merge. I also have a toy hypervisor "lhype" based on this, but it's not ready for mainline. (It seems people expect consoles to do *input* as well as output).
Cheers,
Rusty.
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Rusty RussellThis patch breaks `make headers_check' in mysterious ways:
CHECK include/linux/netfilter_ipv4/ip_conntrack_tcp.h CHECK include/linux/netfilter_ipv4/ip_conntrack_sctp.h CHECK include/linux/netfilter_ipv4/ip_conntrack_protocol.h CHECK include/linux/netfilter_ipv4/ip_conntrack_helper_h323_types.h CHECK include/linux/netfilter_ipv4/ip_conntrack_helper_h323_asn1.h CHECK include/linux/netfilter_ipv4/ip_conntrack_helper.h
make[2]: *** [/usr/src/devel/usr/include/asm/.check.setup.h] Error 1 make[2]: *** Waiting for unfinished jobs....
make[1]: *** [asm-i386] Error 2
make[1]: *** Waiting for unfinished jobs....
make: *** [headers_check] Error 2
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 8 lines · Andrew Morton
This patch breaks `make
headers_check' in mysterious ways:CHECK include/linux/netfilter_ipv4/ip_conntrack_tcp.h CHECK include/linux/netfilter_ipv4/ip_conntrack_sctp.h CHECK include/linux/netfilter_ipv4/ip_conntrack_protocol.h CHECK include/linux/netfilter_ipv4/ip_conntrack_helper_h323_types.h CHECK include/linux/netfilter_ipv4/ip_conntrack_helper_h323_asn1.h CHECK include/linux/netfilter_ipv4/ip_conntrack_helper.h
Quoted 5 lines
make[2]: *** [
/usr/src/devel/usr/include/asm/.check.setup.h] Error 1 make[2]: *** Waiting for unfinished jobs....
make[1]: *** [asm-i386] Error 2
make[1]: *** Waiting for unfinished jobs....
make: *** [headers_check] Error 2
It seems like missing
"header-y += paravirt.h" in the "include/asm-i386/Kbuild".
____
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Rusty RussellQuoted 3 lines · Rusty Russell
Create a
paravirt.hheader for all the critical operations which need to be replaced with hypervisor calls, and include that instead of defining native operations, whenCONFIG_PARAVIRT.
Hmm, did this all ever compile in mainline? I had to do a few merges and in the end i get
/home/lsrc/quilt/linux/kernel/spinlock.c: In function ‘_spin_lock_irqsave’:include2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letter
{standard input}: Assembler messages:
{standard input}:593: Error: undefined symbol paravirt_ops' in operation {standard input}:593: Error: undefined symbol PARAVIRT_irq_enable' in operation {standard input}:605: Error: undefined symbol paravirt_ops' in operation {standard input}:605: Error: undefined symbol PARAVIRT_irq_disable' in operatio n
and lots of new warnings like
/home/lsrc/quilt/linux/arch/i386/kernel/traps.c: In function ‘set_intr_gate’:/home/lsrc/quilt/linux/arch/i386/kernel/traps.c:1165: warning: implicit declarat ion of function ‘_set_gate’ /home/lsrc/quilt/linux/arch/i386/kernel/cpu/common.c: In function ‘_cpu_init’:/home/lsrc/quilt/linux/arch/i386/kernel/cpu/common.c:754: warning: implicit decl aration of function ‘__set_tss_desc’ /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c: In function ‘intel_mach ine_check’:/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.eax’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.ebx’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.ecx’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.edx’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.esi’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.edi’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.ebp’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.esp’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.eflag s’ may be used uninitialized in this function /home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.eip’ may be used uninitialized in this function
This is with i386 defconfig + CONFIG_PARAVIRT
-Andi
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.osdl.org/mailman/listinfo/virtualization
[x86_64] Strange oprofile results on access to per_cpu data
Hi Andi
While doing some oprofile analysis, I got this result on ip_route_input() : one particular instruction seems to spend a lot of cycles.
machine is a dual core 285, 2.6 GHz
/*
* Command line: opannotate -a event:CPU_CLK_UNHALTED/usr/src/linux-2.6.18/vmlinux
*
* Interpretation of command line:
* Output annotated assembly listing with samples
*
* CPU: AMD64 processors, speed 2600.01 MHz (estimated)
* Counted CPU_CLK_UNHALTED events (Cycles outside of halt state) with a unit
mask of 0x00 (No unit mask) count 10000
*/
ffffffff803e9860 <ip_route_input>: /* ip_route_input total: 543098 2.5487 */
/* relevant extract from ip_route_input() */
600 0.0028 :ffffffff803e98b3: mov $0xffffffff806375e0,%rsi
883 0.0041 :ffffffff803e98ba: mov %rax,%rdx
6 2.8e-05 :ffffffff803e98bd: mov %rsi,%rcx
2281 0.0107 :ffffffff803e98c0: cmp 0xf0(%rdx),%r12d
9767 0.0458 :ffffffff803e98c7: jne ffffffff803e98f1
<ip_route_input+0x91>
108 5.1e-04 :ffffffff803e98c9: cmp 0xf4(%rdx),%r14d
41459 0.1946 :ffffffff803e98d0: jne ffffffff803e98f1
<ip_route_input+0x91>
549 0.0026 :ffffffff803e98d2: cmp 0xec(%rdx),%ebx
88604 0.4158 :ffffffff803e98d8: jne ffffffff803e98f1
<ip_route_input+0x91>
478 0.0022 :ffffffff803e98da: mov 0xe8(%rdx),%eax
315 0.0015 :ffffffff803e98e0: test %eax,%eax
241 0.0011 :ffffffff803e98e2: jne ffffffff803e98f1 <ip_route_input+0x91>
248 0.0012 :ffffffff803e98e4: cmp 0xfc(%rdx),%r13b
2314 0.0109 :ffffffff803e98eb: je ffffffff803ea3b3
################ BEGIN
370 0.0017 :ffffffff803e98f1: mov %gs:0x8,%rax
222769 1.0454 :ffffffff803e98fa: incl 0x38(%rcx,%rax,1)
################ END
6 2.8e-05 :ffffffff803e98fe: mov (%rdx),%rdx
833 0.0039 :ffffffff803e9901: test %rdx,%rdx__raw_get_cpu_var(rt_cache_stat).field++ appears to be very expensive
(about 18000 RT_CACHE_STAT_INC(in_hlist_search); are done per second, not an impressive count in fact)
Are segment prefixes that expensive ?
Or is it only the first access to %gs:8 that is doing extra checks ? (because other RT_CACHE_STAT_INC() done in the same function dont have this cost) Or is it the loading of %rcx (done in ffffffff803e98bd) that is stalling ?
I was wondering if avoiding a dependancy would help :
As we dont have TLS support in kernel yet, I was considering trying (just for experimentation) to stick a struct rt_cache_stat in pda, since it avoids one step.
#if defined(RT_CACHE_STAT_IN_PDA)
# define RT_CACHE_STAT_INC(field) add_pda(rt_cache_stat.field, 1) # define addr_of_rt_cache_stat(cpu) &cpu_pda(cpu)->rt_cache_stat #else static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); # define RT_CACHE_STAT_INC(field) (__raw_get_cpu_var(rt_cache_stat).field++) # define addr_of_rt_cache_stat(cpu) &per_cpu(rt_cache_stat, cpu) #endif
so that RT_CACHE_STAT_INC(field) would map to
addl $1,%gs:OFFSET /* no register needed */
Thank you
Eric
Re: [x86_64] Strange oprofile results on access to per_cpu data
Quoted 6 lines · Eric Dumazet
Hi Andi
While doing some oprofile analysis, I got this result on
ip_route_input():
one particular instruction seems to spend a lot of cycles.machine is a dual core 285, 2.6 GHz
Single socket?
Quoted 8 lines
################ BEGIN
370 0.0017 :ffffffff803e98f1: mov %gs:0x8,%rax 222769 1.0454 :ffffffff803e98fa: incl 0x38(%rcx,%rax,1) ################ END 6 2.8e-05 :ffffffff803e98fe: mov (%rdx),%rdx 833 0.0039 :ffffffff803e9901: test %rdx,%rdx__raw_get_cpu_var(rt_cache_stat).field++ appears to be very expensive
First the profile events are not exact. It could be an earlier instruction. The reordering Window is relatively large (~80 macro ops)
But let's assume it is this one.
Weird. Maybe it's a cache miss. Can you check with DATA_CACHE_MISSES ? Or possible a TLB miss, although that's far less likely (L1_AND_L2_DTLB_MISSES)
Quoted 4 lines
(about 18000
RT_CACHE_STAT_INC(in_hlist_search); are done per second, not an impressive count in fact)Are segment prefixes that expensive ?
No, they are supposed to be one cycle only.
Quoted 3 lines
Or is it only the first access to %gs:8 that is doing extra checks ? (because other
RT_CACHE_STAT_INC()done in the same function dont have this cost) is it the loading of %rcx (done inffffffff803e98bd) that is stalling ?
My first guess would be a cache miss of some sort.
Maybe the rest of the code needs enough cache to push this line out. Or since you got dual core with separated caches they are bouncing for some reason (they shouldn't, but maybe there is a bug)
Quoted 3 lines
so that
RT_CACHE_STAT_INC(field)would map toaddl $1,%gs:OFFSET /* no register needed */
I doubt that will make much difference.
-Andi
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Andi KleenQuoted 25 lines · Andi Kleen
Quoted 4 lines · Rusty Russell
Create a
paravirt.hheader for all the critical operations which need to be replaced with hypervisor calls, and include that instead of defining native operations, whenCONFIG_PARAVIRT.Hmm, did this all ever compile in mainline? I had to do a few merges and in the end i get
/home/lsrc/quilt/linux/kernel/spinlock.c: In function ‘_spin_lock_irqsave’:include2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letterinclude2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letter
Quoted 9 lines
include2/asm/spinlock.h:59: error: invalid 'asm': operand number missing after % -letter
{standard input}: Assembler messages:
{standard input}:593: Error: undefined symbolparavirt_ops' in operation {standard input}:593: Error: undefined symbolPARAVIRT_irq_enable' in operation {standard input}:605: Error: undefined symbolparavirt_ops' in operation {standard input}:605: Error: undefined symbolPARAVIRT_irq_disable' in operatio n
Not seeing that here (on 2.6.19-rc2-mm2 with gcc 4.0.2).
Quoted 9 lines
and lots of new warnings like
/home/lsrc/quilt/linux/arch/i386/kernel/traps.c: In function ‘set_intr_gate’:/home/lsrc/quilt/linux/arch/i386/kernel/traps.c:1165: warning: implicit declarat ion of function ‘_set_gate’/home/lsrc/quilt/linux/arch/i386/kernel/cpu/common.c: In function ‘_cpu_init’:/home/lsrc/quilt/linux/arch/i386/kernel/cpu/common.c:754: warning: implicit decl aration of function ‘__set_tss_desc'
Sounds like desc.h got reordered. Somewhere, there was a broken patch once that did this, I thought we fixed that.
Quoted 5 lines
/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c: In function ‘intel_mach
ine_check’:/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.eax’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.ebx’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.ecx’ may be used uninitialized in this function
Quoted 9 lines
/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.edx’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.esi’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.edi’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.ebp’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.esp’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.eflag
s’ may be used uninitialized in this function/home/lsrc/quilt/linux/arch/i386/kernel/cpu/mcheck/p4.c:158: warning: ‘dbg.eip’ may be used uninitialized in this function
Those appear to be valid warnings, with or without paravirt, due to the tacky glued inline oddity of intel_get_extended_msrs.
Zach
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 4 lines
Sounds like
desc.hgot reordered. Somewhere, there was a broken patch once that did this, I thought we fixed that.
I think I got Rusty's latest patches that I found in my mailbox.
I haven't looked at desc.h, but at least processor.h ordering was totally b0rken (e.g. #define __cpuid native_cpuid was after several uses). I fixed that to make at least the CONFIG_PARAVIRT not set case compile.
I can't see how this ever worked either.
Haven't attempted the CONFIG_PARAVIRT case which apparently needs more work (it is currently marked CONFIG_BROKEN)
Can someone double check this is the correct patchkit?
ftp://ftp.frstfloor.org/pub/ak/x86_64/quilt/patches/paravirt*
-Andi
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 10 lines · Andi Kleen
Quoted 3 lines
Sounds like
desc.hgot reordered. Somewhere, there was a broken patch once that did this, I thought we fixed that.I think I got Rusty's latest patches that I found in my mailbox.
I haven't looked at
desc.h, but at leastprocessor.hordering was totally b0rken (e.g. #define__cpuidnative_cpuidwas after several uses). I fixed that to make at least theCONFIG_PARAVIRTnot set case compile.
Quoted 8 lines
I can't see how this ever worked either.
Haven't attempted the
CONFIG_PARAVIRTcase which apparently needs more work (it is currently markedCONFIG_BROKEN)Can someone double check this is the correct patchkit?
ftp://ftp.frstfloor.org/pub/ak/x86_64/quilt/patches/paravirt*
Andi, the patches work against Andrew's tree, and he's merged them in rc4-mm2. There are a few warnings to clean up, but it seems basically sound.
At this point I our think time is better spent on beating those patches up, rather than going back and figuring out why they don't work in your tree.
Sorry,
Rusty.
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 8 lines · Rusty Russell
Andi, the patches work against Andrew's tree, and he's merged them in rc4-mm2. There are a few warnings to clean up, but it seems basically sound.
At this point I our think time is better spent on beating those patches up, rather than going back and figuring out why they don't work in your tree.
This begs the question - should we rebase the paravirt-ops patchset against -rc4-mm2? I almost did it today, but didn't want to stomp on anybody else's toes.
Zach
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 5 lines · Zachary Amsden
Quoted 4 lines · Rusty Russell
Andi, the patches work against Andrew's tree, and he's merged them in rc4-mm2. There are a few warnings to clean up, but it seems basically sound.
Quoted 8 lines
Quoted 4 lines
At this point I our think time is better spent on beating those patches up, rather than going back and figuring out why they don't work in your tree.
This begs the question - should we rebase the paravirt-ops patchset against -rc4-mm2? I almost did it today, but didn't want to stomp on anybody else's toes.
Yes. Andrew has shot me a couple of warnings which people have found, and I'm preparing patches for them. Rebasing will make it easier.
If you're not awake now, I'll do it. If you are, see me on IRC.
Rusty.
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Rusty RussellQuoted 8 lines
Andi, the patches work against Andrew's tree, and he's merged them in rc4-mm2. There are a few warnings to clean up, but it seems basically sound.
At this point I our think time is better spent on beating those patches up, rather than going back and figuring out why they don't work in your tree.
My tree is basically mainline as base. Sure if you don't care about mainline merges we can ignore it there and keep it forever in -mm* until Andrew gets tired of it?
That's a possible strategy, but only if you want to keep it as a mm-only toy forever.
-Andi
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 9 lines · On Sun, 5 Nov 2006 06:46:15 +0100 Andi Kleen <[email protected]> wrote:
Quoted 8 lines
Andi, the patches work against Andrew's tree, and he's merged them in rc4-mm2. There are a few warnings to clean up, but it seems basically sound.
At this point I our think time is better spent on beating those patches up, rather than going back and figuring out why they don't work in your tree.
Quoted 7 lines
My tree is basically mainline as base. Sure if you don't care about mainline merges we can ignore it there and keep it forever in -mm* until Andrew gets tired of it?
That's a possible strategy, but only if you want to keep it as a mm-only toy forever.
They're in my regular list-of-thing-to-spam-maintainers-with, so we can transfer them as-is next week sometime.
It would be better to sort out the various warnings and any other nasties first though.
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Andi KleenQuoted 8 lines · Andi Kleen
Quoted 7 lines
Andi, the patches work against Andrew's tree, and he's merged them in rc4-mm2. There are a few warnings to clean up, but it seems basically sound.
At this point I our think time is better spent on beating those patches up, rather than going back and figuring out why they don't work in your tree.
Quoted 6 lines
My tree is basically mainline as base. Sure if you don't care about mainline merges we can ignore it there and keep it forever in -mm* until Andrew gets tired of it?
That's a possible strategy, but only if you want to keep it as a mm-only toy forever.
Andi, it's been simpler for us to get the code into Andrew's tree, in nice bit-size pieces. We've had trouble every time we've tried to get stuff into your tree. In addition, Andrew's tree gives the code exposure and testing.
If Andrew says we have to get those patches into mainline through you, then I'll spend all that time re-spinning the patches for you from the -mm tree until they go in. It doesn't seem like a good use of anyone's time though.
Rusty.
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
Quoted 3 lines
If Andrew says we have to get those patches into mainline through you,
Well I'm mainline in this case.
Quoted 2 lines
then I'll spend all that time re-spinning the patches for you from the -mm tree until they go in.
I got it to compile now with this patch (+ one additional patch that is folded in). It then goes through kernel initialization and then init gets killed with "Inconsistency detected by rtld.c:1250:
Assertation ph_vaddr == _rtld_local.dl_sysinfo_vdso failed"
It looks like some of the ifdefs were placed completely wrong and in addition you were missing a patch to include asm/offset.h everywhere as assembly (I patched around that). And two macros were apparently never compiled in their current form.
But it seems it is dependent on even more -mm* magic than just that. If you can identify the missing patches that make init's rtld work again that would be useful.
-Andi
Get paravirt ops to compile
TBD should be folded into the original patches
Unfortunately still doesn't boot.
include/asm-i386/desc.h +3 -3
@@ -92,6 +92,9 @@ static inline void write_dt_entry(void *#define write_gdt_entry (dt, entry, a, b) write_dt_entry(dt, entry, a, b)#define write_idt_entry (dt, entry, a, b) write_dt_entry(dt, entry, a, b) +#define set_ldt native_set_ldt +#endif /* CONFIG_PARAVIRT */ +static inline void _set_gate (int gate,unsigned int type,void *addr,unsigned short seg) {__u32 a, b;
@@ -108,9 +111,6 @@ static inline void __set_tss_desc(unsignwrite_gdt_entry (get_cpu_gdt_table (cpu), entry, a, b); } -#define set_ldt native_set_ldt -#endif /* CONFIG_PARAVIRT */ - static inline fastcall void native_set_ldt(const void *addr,unsigned int entries) {
include/asm-i386/paravirt.h +10 -6
@@ -454,16 +454,20 @@ static inline unsigned long __raw_local_ return f; } -#define CLI_STRING paravirt_alt("pushl %ecx; pushl %edx;" \ - "call *paravirt_ops+PARAVIRT_irq_disable;" \ - "popl %edx; popl %ecx", \ +#define CLI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \ + "call *paravirt_ops+%c[irq_disable];" \ + "popl %%edx; popl %%ecx", \ PARAVIRT_IRQ_DISABLE, CLBR_EAX) -#define STI_STRING paravirt_alt("pushl %ecx; pushl %edx;" \ - "call *paravirt_ops+PARAVIRT_irq_enable;" \ - "popl %edx; popl %ecx", \ +#define STI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \ + "call *paravirt_ops+%c[irq_enable];" \ + "popl %%edx; popl %%ecx", \ PARAVIRT_IRQ_ENABLE, CLBR_EAX)#define CLI_STI_CLOBBERS , "%eax" +#define CLI_STI_INPUT_ARGS \ + , \ + [irq_disable] "i" (offsetof(struct paravirt_ops, irq_disable)), \ + [irq_enable] "i" (offsetof(struct paravirt_ops, irq_enable))#else /* __ASSEMBLY__ */
include/asm-i386/spinlock.h +8 -6
@@ -13,6 +13,7 @@#define CLI_STRING "cli"#define STI_STRING "sti"#define CLI_STI_CLOBBERS +#define CLI_STI_INPUT_ARGS #endif /* CONFIG_PARAVIRT */ /*
@@ -58,26 +59,27 @@ static inline void __raw_spin_lock_flags { asmvolatile ("\n1:\t" -LOCK_PREFIX " ; decb %0\n\t" +LOCK_PREFIX " ; decb %[slock]\n\t" "jns 5f\n" "2:\t" -"testl $0x200, %1\n\t" +"testl $0x200, %[flags]\n\t" "jz 4f\n\t" STI_STRING "\n" "3:\t" "rep;nop\n\t" -"cmpb $0, %0\n\t" +"cmpb $0, %[slock]\n\t" "jle 3b\n\t" CLI_STRING "\n\t" "jmp 1b\n" "4:\t" "rep;nop\n\t" -"cmpb $0, %0\n\t" +"cmpb $0, %[slock]\n\t" "jg 1b\n\t" "jmp 4b\n" "5:\n\t" - :"+m" (lock->slock) - :"r" (flags) + : [slock]"+m" (lock->slock) + : [flags]"r" (flags) + CLI_STI_INPUT_ARGS :"memory" CLI_STI_CLOBBERS ); }#endif
include/asm-i386/processor.h +2 -0
@@ -511,6 +511,7 @@ static inline void load_esp0(struct tss_ wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0); } } +#endif #define start_thread (regs, new_eip, new_esp) do { \ __asm__("movl %0,%%fs": :"r" (0)); \
@@ -524,6 +525,7 @@ static inline void load_esp0(struct tss_ regs->esp = new_esp; \ }while (0 ) +#ifndef CONFIG_PARAVIRT/* * These special macros can be used to get or set a debugging register */
Re: [PATCH 1/7] paravirtualization: header and stubs for paravirtualizing critical operations
In reply to Rusty RussellQuoted 9 lines · Rusty Russell
Create a
paravirt.hheader for all the critical operations which need to be replaced with hypervisor calls, and include that instead of defining native operations, whenCONFIG_PARAVIRT.This patch does the dumbest possible replacement of paravirtualized
instructions: calls through a "paravirt_ops" structure. Currently these are function implementations of native hardware: hypervisors will override the ops structure with their own variants.
[snip]
Quoted 2 lines
arch/i386/kernel/paravirt.c new file +2
-0+struct paravirt_ops paravirt_ops = { + .name ="bare hardware" ,
[snip]
Quoted 2 lines
arch/i386/kernel/paravirt.c new file +2
-0+ .get_wallclock = native_get_wallclock, + .set_wallclock = native_set_wallclock,
[snip]
Quoted 29 lines
include/asm-i386/time.h new file +26
-0@@ -0,0 +1,41 @@ +#ifndef _ASMi386_TIME_H +#define _ASMi386_TIME_H + +#include <linux/efi.h> +#include "mach_time.h" + +static inline unsigned long native_get_wallclock (void ) +{ +unsigned long retval; + +if (efi_enabled) + retval =efi_get_time (); +else + retval =mach_get_cmos_time (); + +return retval; +} + +static inline int native_set_wallclock (unsigned long nowtime) +{ +int retval; + +if (efi_enabled) + retval =efi_set_rtc_mmss (nowtime); +else + retval =mach_set_rtc_mmss (nowtime);
include/asm-i386/time.h line 36
Quoted 10 lines
include/asm-i386/time.h new file +10
-0+ +return retval; +} + +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else /* !CONFIG_PARAVIRT */ + +#define get_wallclock () native_get_wallclock() +#define set_wallclock (x) native_set_wallclock(x)
Could a better name then "get/set_wallclock" be used here? Its too vague and would be easily confused with do_set/gettimeofday() functions.
My suggestion would be to use "persistent_clock" to describe the battery-backed CMOS/hardware clock. (I assume that is what you intend this paravirt_op to be, rather then get the high-resolution system timeofday)
thanks
-john
PATCH 7/7 Add mmu virtualization to paravirt-ops.
In reply to Chris WrightAdd the three bare TLB accessor functions to paravirt-ops. Most amusingly, flush_tlb is redefined on SMP, so I can't call the paravirt op flush_tlb. Instead, I chose to indicate the actual flush type, kernel (global) vs. user (non-global). Global in this sense means using the global bit in the page table entry, which makes TLB entries persistent across CR3 reloads, not global as in the SMP sense of invoking remote shootdowns, so the term is confusingly overloaded.
--- arch/i386/kernel/paravirt.c | 109 ++++++++++++++++++++++++++++++++++++++ arch/i386/mm/boot_ioremap.c | 1 include/asm-i386/paravirt.h | 75 ++++++++++++++++++++++++++ include/asm-i386/pgtable-2level.h | 5 + include/asm-i386/pgtable-3level.h | 42 +++++++------- include/asm-i386/pgtable.h | 4 + include/asm-i386/tlbflush.h | 18 ++++-- 7 files changed, 227 insertions(+), 27 deletions(-)
arch/i386/kernel/paravirt.c +109 -0
@@ -30,6 +30,7 @@#include <asm/delay.h> #include <asm/fixmap.h> #include <asm/apic.h> +#include <asm/tlbflush.h> /* nop stub */ static void native_nop (void )
@@ -404,6 +405,97 @@ static fastcall unsigned long native_api }#endif /* CONFIG_X86_LOCAL_APIC */ +static fastcall void native_flush_tlb(void) +{ +__native_flush_tlb (); +} + +/* +* Global pages have to be flushed a bit differently. Not a real +* performance problem because this does not happen often. +*/ +static fastcall void native_flush_tlb_global(void) +{ +__native_flush_tlb_global (); +} + +static fastcall void native_flush_tlb_single(u32 addr) +{ +__native_flush_tlb_single (addr); +} + +#ifndef CONFIG_X86_PAE +static fastcall void native_set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; +} + +static fastcall void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; +} + +static fastcall void native_set_pmd(pmd_t *pmdp, pmd_t pmdval) +{ + *pmdp = pmdval; +} + +#else /* CONFIG_X86_PAE */ + +static fastcall void native_set_pte(pte_t *ptep, pte_t pte) +{ + ptep->pte_high = pte.pte_high; +smp_wmb (); + ptep->pte_low = pte.pte_low; +} + +static fastcall void native_set_pte_at(struct mm_struct *mm, u32 addr, pte_t *ptep, pte_t pte) +{ + ptep->pte_high = pte.pte_high; +smp_wmb (); + ptep->pte_low = pte.pte_low; +} + +static fastcall void native_set_pte_present(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) +{ + ptep->pte_low =0 ; +smp_wmb (); + ptep->pte_high = pte.pte_high; +smp_wmb (); + ptep->pte_low = pte.pte_low; +} + +static fastcall void native_set_pte_atomic(pte_t *ptep, pte_t pteval) +{ +set_64bit ((unsigned long long *)ptep,pte_val (pteval)); +} + +static fastcall void native_set_pmd(pmd_t *pmdp, pmd_t pmdval) +{ +set_64bit ((unsigned long long *)pmdp,pmd_val (pmdval)); +} + +static fastcall void native_set_pud(pud_t *pudp, pud_t pudval) +{ + *pudp = pudval; +} + +static fastcall void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + ptep->pte_low =0 ; +smp_wmb (); + ptep->pte_high =0 ; +} + +static fastcall void native_pmd_clear(pmd_t *pmd) +{ +u32 *tmp = (u32 *)pmd; + *tmp =0 ; +smp_wmb (); + *(tmp +1 ) =0 ; +} +#endif /* CONFIG_X86_PAE */ +/* These are in entry.S */ extern fastcall void native_iret(void); extern fastcall void native_irq_enable_sysexit(void);
@@ -480,6 +572,23 @@ struct paravirt_ops paravirt_ops = { .apic_read = native_apic_read, #endif + .flush_tlb_user = native_flush_tlb, + .flush_tlb_kernel = native_flush_tlb_global, + .flush_tlb_single = native_flush_tlb_single, + + .set_pte = native_set_pte, + .set_pte_at = native_set_pte_at, + .set_pmd = native_set_pmd, + .pte_update = (void *)native_nop, + .pte_update_defer = (void *)native_nop, +#ifdef CONFIG_X86_PAE + .set_pte_atomic = native_set_pte_atomic, + .set_pte_present = native_set_pte_present, + .set_pud = native_set_pud, + .pte_clear = native_pte_clear, + .pmd_clear = native_pmd_clear, +#endif + .irq_enable_sysexit = native_irq_enable_sysexit, .iret = native_iret, };
arch/i386/mm/boot_ioremap.c +1 -0
@@ -16,6 +16,7 @@*/ #undef CONFIG_X86_PAE +#undef CONFIG_PARAVIRT#include <asm/page.h> #include <asm/pgtable.h> #include <asm/tlbflush.h>
include/asm-i386/paravirt.h +75 -0
99 lines not on this page · raw message
include/asm-i386/pgtable-2level.h +4 -1
@@ -11,11 +11,14 @@* within a page table are directly modified. Thus, the following * hook is made available. */ +#ifndef CONFIG_PARAVIRT #define set_pte (pteptr, pteval) (*(pteptr) = pteval)#define set_pte_at (mm,addr,ptep,pteval) set_pte(ptep,pteval) +#define set_pmd (pmdptr, pmdval) (*(pmdptr) = (pmdval)) +#endif +#define set_pte_atomic (pteptr, pteval) set_pte(pteptr,pteval)#define set_pte_present (mm,addr,ptep,pteval) set_pte_at(mm,addr,ptep,pteval) -#define set_pmd (pmdptr, pmdval) (*(pmdptr) = (pmdval))#define pte_clear (mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0)#define pmd_clear (xp) do { set_pmd(xp, __pmd(0)); } while (0)
include/asm-i386/pgtable-3level.h +22 -20
66 lines not on this page · raw message
include/asm-i386/pgtable.h +3 -1
@@ -15,6 +15,7 @@#include <asm/processor.h> #include <asm/fixmap.h> #include <linux/threads.h> +#include <asm/paravirt.h> #ifndef _I386_BITOPS_H#include <asm/bitops.h>
@@ -246,6 +247,7 @@ static inline pte_t pte_mkhuge(pte_t pte# include <asm/pgtable-2level.h> #endif +#ifndef CONFIG_PARAVIRT /* * Rules for using pte_update - it must be called after any PTE update which * has not been done using the set_pte / clear_pte interfaces. It is used by
@@ -261,7 +263,7 @@ static inline pte_t pte_mkhuge(pte_t pte*/ #define pte_update (mm, addr, ptep) do { } while (0)#define pte_update_defer (mm, addr, ptep) do { } while (0) - +#endif /* * We only update the dirty/accessed state if we set
include/asm-i386/tlbflush.h +13 -5
@@ -4,7 +4,15 @@#include <linux/mm.h> #include <asm/processor.h> -#define __flush_tlb () \ +#ifdef CONFIG_PARAVIRT +#include <asm/paravirt.h> +#else +#define __flush_tlb () __native_flush_tlb() +#define __flush_tlb_global () __native_flush_tlb_global() +#define __flush_tlb_single (addr) __native_flush_tlb_single(addr) +#endif + +#define __native_flush_tlb () \ do { \ unsigned int tmpreg; \ \
@@ -19,7 +27,7 @@* Global pages have to be flushed a bit differently. Not a real * performance problem because this does not happen often. */ -#define __flush_tlb_global () \ +#define __native_flush_tlb_global () \ do { \ unsigned int tmpreg, cr4, cr4_orig; \ \
@@ -36,6 +44,9 @@ : "memory"); \ } while (0) +#define __native_flush_tlb_single (addr) \ + __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory") +# define __flush_tlb_all () \ do { \ if (cpu_has_pge) \
@@ -46,9 +57,6 @@#define cpu_has_invlpg (boot_cpu_data.x86 > 3) -#define __flush_tlb_single (addr) \ - __asm__ __volatile__("invlpg (%0)" ::"r" (addr) : "memory") -#ifdef CONFIG_X86_INVLPG # define __flush_tlb_one (addr) __flush_tlb_single(addr)#else
--