KVM: Add asynchronous page fault for PV guest.

PATCH Superseded Gleb Natapov 4 participants 30 messages Nov 23, 2009 – Nov 30, 2009

Subsystems (13)

Memory management x86 KVM Library code +9 more subsystems

lore

Share & tools

lore

Gleb Natapov

author

2009-11-23 14:05 UTC #0 lore raw

v2 cover KVM: Add asynchronous page fault for PV guest.

KVM virtuaize guest memory by means of shadow pages or HW assistance like NPT/EPT. Not all memory used by a guest is mapped into the guest address space or even present in a host memory at any given time. When vcpu tries to access memory page that is not mapped into guest address space KVM is notified about it. KVM maps the page into guest address space and resumes vcpu execution. If the page is swapped out from host memory vcpu execution is suspended till page is not swapped into the memory again. This is inefficient since vcpu can do other work (run other task or serve interrupts) while page gets swapped in.

To overcome this inefficient this patch series implements "asynchronous page fault" for paravirtualized KVM guests. If a page that vcpu is trying to access is swapped out KVM sends async PF to the vcpu and continues vcpu execution. Requested page is swapped in by another thread in parallel. When vcpu gets async PF it puts the task that faulted to sleep until "wake up" interrupt is delivered. When page is brought to host memory KVM sends "wake up" interrupt and the guest task resumes execution.

Changes:
v1->v2
Use MSR instead of hypercall.
Move most of the code into arch independent place. halt inside a guest instead of doing "wait for page" hypercall if
preemption is disabled.

Gleb Natapov (12):
Move kvm_smp_prepare_boot_cpu() from kvmclock.c to kvm.c. Add PV MSR to enable asynchronous page faults delivery. Add async PF initialization to PV guest.
Add "handle page fault" PV helper.
Handle asynchronous page fault in a PV guest.
Export __get_user_pages_fast.
Add get_user_pages() variant that fails if major fault is required. Inject asynchronous page fault into a guest if page is swapped out. Retry fault before vmentry
Maintain preemptability count even for !CONFIG_PREEMPT kernels Handle async PF in non preemptable context.
Send async PF when guest is not in userspace too.

 arch/x86/include/asm/kvm_host.h       |   22 +++-
 arch/x86/include/asm/kvm_para.h       |   11 ++
 arch/x86/include/asm/paravirt.h       |    7 +
 arch/x86/include/asm/paravirt_types.h |    4 +
 arch/x86/kernel/kvm.c                 |  215 +++++++++++++++++++++++++++++++
 arch/x86/kernel/kvmclock.c            |   13 +--
 arch/x86/kernel/paravirt.c            |    8 +
 arch/x86/kernel/paravirt_patch_32.c   |    8 +
 arch/x86/kernel/paravirt_patch_64.c   |    7 +
 arch/x86/kernel/smpboot.c             |    3 +
 arch/x86/kvm/Kconfig                  |    2 +
 arch/x86/kvm/mmu.c                    |   46 ++++++-
 arch/x86/kvm/paging_tmpl.h            |   50 +++++++-
 arch/x86/kvm/x86.c                    |   86 ++++++++++++-
 arch/x86/mm/fault.c                   |    3 +
 arch/x86/mm/gup.c                     |    2 +
 fs/ncpfs/mmap.c                       |    2 +
 include/linux/hardirq.h               |   14 +--
 include/linux/kvm.h                   |    1 +
 include/linux/kvm_host.h              |   27 ++++
 include/linux/kvm_para.h              |    2 +
 include/linux/mm.h                    |    5 +
 include/linux/preempt.h               |   22 +++-
 include/linux/sched.h                 |    4 -
 include/trace/events/kvm.h            |   60 +++++++++
 kernel/sched.c                        |    6 -
 lib/kernel_lock.c                     |    1 +
 mm/filemap.c                          |    3 +
 mm/memory.c                           |   31 ++++-
 mm/shmem.c                            |    8 +-
 virt/kvm/Kconfig                      |    3 +
 virt/kvm/kvm_main.c                   |  227 ++++++++++++++++++++++++++++++++-
 32 files changed, 846 insertions(+), 57 deletions(-)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:05 UTC #1 lore raw

PATCH v2 1/12 Move kvm_smp_prepare_boot_cpu() from kvmclock.c to kvm.c.

Async PF also needs to hook into smp_prepare_boot_cpu so move the hook into generic code.

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/include/asm/kvm_para.h |    1 +
 arch/x86/kernel/kvm.c           |   11 +++++++++++
 arch/x86/kernel/kvmclock.c      |   13 +------------
 3 files changed, 13 insertions(+), 12 deletions(-)

arch/x86/include/asm/kvm_para.h +1 -0

index c584076..5f580f2 100644
@@ -51,6 +51,7 @@ struct kvm_mmu_op_release_pt {
 #include <asm/processor.h>
 
 extern void kvmclock_init(void);
+extern int kvm_register_clock(char *txt);
 
 
 /* This instruction is vmcall.  On non-VT architectures, it will generate a

arch/x86/kernel/kvm.c +11 -0

index 63b0ec8..e6db179 100644
@@ -231,10 +231,21 @@ static void __init paravirt_ops_setup(void)
 #endif
 }
 
+#ifdef CONFIG_SMP
+static void __init kvm_smp_prepare_boot_cpu(void)
+{
+	WARN_ON(kvm_register_clock("primary cpu clock"));
+	native_smp_prepare_boot_cpu();
+}
+#endif
+
 void __init kvm_guest_init(void)
 {
 	if (!kvm_para_available())
 		return;
 
 	paravirt_ops_setup();
+#ifdef CONFIG_SMP
+	smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
+#endif
 }

arch/x86/kernel/kvmclock.c +1 ~~-12~~

index feaeb0d..6ab9622 100644
@@ -122,7 +122,7 @@ static struct clocksource kvm_clock = {
 	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
 };
 
-static int kvm_register_clock(char *txt)
+int kvm_register_clock(char *txt)
 {
 	int cpu = smp_processor_id();
 	int low, high;

@@ -146,14 +146,6 @@ static void __cpuinit kvm_setup_secondary_clock(void)
 }
 #endif
 
-#ifdef CONFIG_SMP
-static void __init kvm_smp_prepare_boot_cpu(void)
-{
-	WARN_ON(kvm_register_clock("primary cpu clock"));
-	native_smp_prepare_boot_cpu();
-}
-#endif
-
 /*
  * After the clock is registered, the host will keep writing to the
  * registered memory location. If the guest happens to shutdown, this memory

@@ -192,9 +184,6 @@ void __init kvmclock_init(void)
 		x86_cpuinit.setup_percpu_clockev =
 			kvm_setup_secondary_clock;
 #endif
-#ifdef CONFIG_SMP
-		smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;
-#endif
 		machine_ops.shutdown  = kvm_shutdown;
 #ifdef CONFIG_KEXEC
 		machine_ops.crash_shutdown  = kvm_crash_shutdown;

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:05 UTC #2 lore raw

PATCH v2 2/12 Add PV MSR to enable asynchronous page faults delivery.

In reply to Gleb Natapov

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |    3 ++
 arch/x86/include/asm/kvm_para.h |    2 +
 arch/x86/kvm/x86.c              |   42 +++++++++++++++++++++++++++++++++++++-
 include/linux/kvm.h             |    1 +
 4 files changed, 46 insertions(+), 2 deletions(-)

arch/x86/include/asm/kvm_host.h +3 -0

index 06e0856..9598e85 100644
@@ -374,6 +374,9 @@ struct kvm_vcpu_arch {
 	/* used for guest single stepping over the given code position */
 	u16 singlestep_cs;
 	unsigned long singlestep_rip;
+
+	u32 __user *apf_data;
+	u64 apf_msr_val;
 };
 
 struct kvm_mem_alias {

arch/x86/include/asm/kvm_para.h +2 -0

index 5f580f2..222d5fd 100644
@@ -15,9 +15,11 @@
 #define KVM_FEATURE_CLOCKSOURCE		0
 #define KVM_FEATURE_NOP_IO_DELAY	1
 #define KVM_FEATURE_MMU_OP		2
+#define KVM_FEATURE_ASYNC_PF		3
 
 #define MSR_KVM_WALL_CLOCK  0x11
 #define MSR_KVM_SYSTEM_TIME 0x12
+#define MSR_KVM_ASYNC_PF_EN 0x13
 
 #define KVM_MAX_MMU_OP_BATCH           32

arch/x86/kvm/x86.c +40 -2

85 lines not on this page · raw message

include/linux/kvm.h +1 -0

index 92045a9..6af1c99 100644
@@ -492,6 +492,7 @@ struct kvm_ioeventfd {
 #ifdef __KVM_HAVE_VCPU_EVENTS
 #define KVM_CAP_VCPU_EVENTS 41
 #endif
+#define KVM_CAP_ASYNC_PF 42
 
 #ifdef KVM_CAP_IRQ_ROUTING

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Avi Kivity

2009-11-25 12:32 UTC #3 lore raw

Re: [PATCH v2 02/12] Add PV MSR to enable asynchronous page faults delivery.

Quoted 8 lines · Gleb Natapov

Signed-off-by Gleb Natapov<[email protected]>

---
  arch/x86/include/asm/kvm_host.h |    3 ++
  arch/x86/include/asm/kvm_para.h |    2 +
  arch/x86/kvm/x86.c              |   42 +++++++++++++++++++++++++++++++++++++-
  include/linux/kvm.h             |    1 +
  4 files changed, 46 insertions(+), 2 deletions(-)

Quoted 4 lines

#define MSR_KVM_WALL_CLOCK 0x11
#define MSR_KVM_SYSTEM_TIME 0x12
+#define MSR_KVM_ASYNC_PF_EN 0x13

Please use MSRs from the range 0x4b564dxx. The numbers below are reserved by Intel (and in fact used by the old Pentiums).

Need documentation for the new MSR, say in Documentation/kvm/msr.txt.

Quoted 8 lines

arch/x86/kvm/x86.c +7 -0

+static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
+{
+	u64 gpa = data&  ~0x3f;
+	int offset = offset_in_page(gpa);
+	unsigned long addr;
+
+	addr = gfn_to_hva(vcpu->kvm, gpa>>  PAGE_SHIFT);

arch/x86/kvm/x86.c out: · line 968

Quoted 10 lines

arch/x86/kvm/x86.c +10 -0

+	if (kvm_is_error_hva(addr))
+		return 1;
+
+	vcpu->arch.apf_data = (u32 __user*)(addr + offset);
+
+	/* check if address is mapped */
+	if (get_user(offset, vcpu->arch.apf_data)) {
+		vcpu->arch.apf_data = NULL;
+		return 1;
+	}

What if the memory slot arrangement changes? This needs to be revalidated (and gfn_to_hva() called again).

Quoted 7 lines

arch/x86/kvm/x86.c +3 -0

+	return 0;
+}
+
  int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  {
  	switch (msr) {

arch/x86/kvm/x86.c int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) · line 1057

Quoted 9 lines

arch/x86/kvm/x86.c +5 -0

@@ -1029,6 +1049,14 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
  		kvm_request_guest_time_update(vcpu);
  		break;
  	}
+	case MSR_KVM_ASYNC_PF_EN:
+		vcpu->arch.apf_msr_val = data;
+		if (data&  1) {
+			if (kvm_pv_enable_async_pf(vcpu, data))
+				return 1;

Need to check before setting the msr value, so subsequent reads return the old value.

arch/x86/kvm/x86.c int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)

Quoted 3 lines

arch/x86/kvm/x86.c +2 -0

+		} else
+			vcpu->arch.apf_data = NULL;

Need to check that bits 1:5 are zero. I think it's cleaner to move all of the code to kvm_pv_enable_async_pf(), to have everything in one place.

Signature · 7 lines

error compiling committee.c: too many arguments to function

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:05 UTC #4 lore raw

PATCH v2 3/12 Add async PF initialization to PV guest.

In reply to Gleb Natapov

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/include/asm/kvm_para.h |    5 ++++
 arch/x86/kernel/kvm.c           |   49 +++++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/smpboot.c       |    3 ++
 include/linux/kvm_para.h        |    2 +
 4 files changed, 59 insertions(+), 0 deletions(-)

arch/x86/include/asm/kvm_para.h +5 -0

index 222d5fd..d7d7079 100644
@@ -49,6 +49,11 @@ struct kvm_mmu_op_release_pt {
 	__u64 pt_phys;
 };
 
+struct kvm_vcpu_pv_apf_data {
+	__u32 reason;
+	__u32 enabled;
+};
+
 #ifdef __KERNEL__
 #include <asm/processor.h>

arch/x86/kernel/kvm.c +49 -0

79 lines not on this page · raw message

arch/x86/kernel/smpboot.c +3 -0

index 565ebc6..5599098 100644
@@ -65,6 +65,7 @@
 #include <asm/setup.h>
 #include <asm/uv/uv.h>
 #include <linux/mc146818rtc.h>
+#include <linux/kvm_para.h>
 
 #include <asm/smpboot_hooks.h>

@@ -321,6 +322,8 @@ notrace static void __cpuinit start_secondary(void *unused)
 	ipi_call_unlock();
 	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
 
+	kvm_guest_cpu_init();
+
 	/* enable local interrupts */
 	local_irq_enable();

include/linux/kvm_para.h +2 -0

index d731092..4c8a2e6 100644
@@ -26,8 +26,10 @@
 #ifdef __KERNEL__
 #ifdef CONFIG_KVM_GUEST
 void __init kvm_guest_init(void);
+void __cpuinit kvm_guest_cpu_init(void);
 #else
 #define kvm_guest_init() do { } while (0)
+#define kvm_guest_cpu_init() do { } while (0)
 #endif
 
 static inline int kvm_para_has_feature(unsigned int feature)

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:05 UTC #5 lore raw

PATCH v2 4/12 Add "handle page fault" PV helper.

In reply to Gleb Natapov

Allow paravirtualized guest to do special handling for some page faults.

Ingo's concerns not yet addressed here. What was the conclusion of previous discussion?

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/include/asm/paravirt.h       |    7 +++++++
 arch/x86/include/asm/paravirt_types.h |    4 ++++
 arch/x86/kernel/paravirt.c            |    8 ++++++++
 arch/x86/kernel/paravirt_patch_32.c   |    8 ++++++++
 arch/x86/kernel/paravirt_patch_64.c   |    7 +++++++
 arch/x86/mm/fault.c                   |    3 +++
 6 files changed, 37 insertions(+), 0 deletions(-)

arch/x86/include/asm/paravirt.h +7 -0

index efb3899..5203da1 100644
@@ -6,6 +6,7 @@
 #ifdef CONFIG_PARAVIRT
 #include <asm/pgtable_types.h>
 #include <asm/asm.h>
+#include <asm/ptrace.h>
 
 #include <asm/paravirt_types.h>

@@ -710,6 +711,12 @@ static inline void arch_end_context_switch(struct task_struct *next)
 	PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
 }
 
+static inline int arch_handle_page_fault(struct pt_regs *regs,
+					 unsigned long error_code)
+{
+	return PVOP_CALL2(int, pv_cpu_ops.handle_pf, regs, error_code);
+}
+
 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
 static inline void arch_enter_lazy_mmu_mode(void)
 {

arch/x86/include/asm/paravirt_types.h +4 -0

index 9357473..bcc39b3 100644
@@ -186,6 +186,7 @@ struct pv_cpu_ops {
 
 	void (*start_context_switch)(struct task_struct *prev);
 	void (*end_context_switch)(struct task_struct *next);
+	int (*handle_pf)(struct pt_regs *regs, unsigned long error_code);
 };
 
 struct pv_irq_ops {

@@ -385,6 +386,7 @@ extern struct pv_lock_ops pv_lock_ops;
 unsigned paravirt_patch_nop(void);
 unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
 unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
+unsigned paravirt_patch_ret_0(void *insnbuf, unsigned len);
 unsigned paravirt_patch_ignore(unsigned len);
 unsigned paravirt_patch_call(void *insnbuf,
 			     const void *target, u16 tgt_clobbers,

@@ -676,8 +678,10 @@ void paravirt_leave_lazy_mmu(void);
 void _paravirt_nop(void);
 u32 _paravirt_ident_32(u32);
 u64 _paravirt_ident_64(u64);
+unsigned long _paravirt_ret_0(void);
 
 #define paravirt_nop	((void *)_paravirt_nop)
+#define paravirt_ret_0  ((void *)_paravirt_ret_0)
 
 /* These all sit in the .parainstructions section to tell us what to patch. */
 struct paravirt_patch_site {

arch/x86/kernel/paravirt.c +8 -0

index 1b1739d..7d8f37b 100644
@@ -54,6 +54,11 @@ u64 _paravirt_ident_64(u64 x)
 	return x;
 }
 
+unsigned long _paravirt_ret_0(void)
+{
+	return 0;
+}
+
 void __init default_banner(void)
 {
 	printk(KERN_INFO "Booting paravirtualized kernel on %s\n",

@@ -154,6 +159,8 @@ unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
 		ret = paravirt_patch_ident_32(insnbuf, len);
 	else if (opfunc == _paravirt_ident_64)
 		ret = paravirt_patch_ident_64(insnbuf, len);
+	else if (opfunc == _paravirt_ret_0)
+		ret = paravirt_patch_ret_0(insnbuf, len);
 
 	else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) ||
 		 type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_sysexit) ||

@@ -380,6 +387,7 @@ struct pv_cpu_ops pv_cpu_ops = {
 
 	.start_context_switch = paravirt_nop,
 	.end_context_switch = paravirt_nop,
+	.handle_pf = paravirt_ret_0,
 };
 
 struct pv_apic_ops pv_apic_ops = {

arch/x86/kernel/paravirt_patch_32.c +8 -0

index d9f32e6..de006b1 100644
@@ -12,6 +12,8 @@ DEF_NATIVE(pv_mmu_ops, read_cr3, "mov %cr3, %eax");
 DEF_NATIVE(pv_cpu_ops, clts, "clts");
 DEF_NATIVE(pv_cpu_ops, read_tsc, "rdtsc");
 
+DEF_NATIVE(, mov0, "xor %eax, %eax");
+
 unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len)
 {
 	/* arg in %eax, return in %eax */

@@ -24,6 +26,12 @@ unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
 	return 0;
 }
 
+unsigned paravirt_patch_ret_0(void *insnbuf, unsigned len)
+{
+	return paravirt_patch_insns(insnbuf, len,
+				    start__mov0, end__mov0);
+}
+
 unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
 		      unsigned long addr, unsigned len)
 {

arch/x86/kernel/paravirt_patch_64.c +7 -0

index 3f08f34..d685e7d 100644
@@ -21,6 +21,7 @@ DEF_NATIVE(pv_cpu_ops, swapgs, "swapgs");
 
 DEF_NATIVE(, mov32, "mov %edi, %eax");
 DEF_NATIVE(, mov64, "mov %rdi, %rax");
+DEF_NATIVE(, mov0, "xor %rax, %rax");
 
 unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len)
 {

@@ -34,6 +35,12 @@ unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len)
 				    start__mov64, end__mov64);
 }
 
+unsigned paravirt_patch_ret_0(void *insnbuf, unsigned len)
+{
+	return paravirt_patch_insns(insnbuf, len,
+				    start__mov0, end__mov0);
+}
+
 unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
 		      unsigned long addr, unsigned len)
 {

arch/x86/mm/fault.c +3 -0

index f4cee90..14707dc 100644
@@ -952,6 +952,9 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
 	int write;
 	int fault;
 
+	if (arch_handle_page_fault(regs, error_code))
+		return;
+
 	tsk = current;
 	mm = tsk->mm;

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Peter Zijlstra

maintainer

2009-11-23 15:32 UTC #6 lore raw

Re: [PATCH v2 04/12] Add "handle page fault" PV helper.

Quoted 5 lines · Gleb Natapov

arch/x86/mm/fault.c +0 -0
index f4cee90..14707dc 100644

arch/x86/mm/fault.c do_page_fault(struct pt_regs *regs, unsigned long error_code) · line 960

Quoted 9 lines

arch/x86/mm/fault.c +3 -0

@@ -952,6 +952,9 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
 	int write;
 	int fault;
 
+	if (arch_handle_page_fault(regs, error_code))
+		return;
+
 	tsk = current;
 	mm = tsk->mm;

That's a bit daft, the pagefault handler is already arch specific, so you're placing an arch_ hook into arch code, that doesn't make sense.

Gleb Natapov

author

2009-11-23 14:06 UTC #7 lore raw

PATCH v2 5/12 Handle asynchronous page fault in a PV guest.

In reply to Gleb Natapov

Asynchronous page fault notifies vcpu that page it is trying to access is swapped out by a host. In response guest puts a task that caused the fault to sleep until page is swapped in again. When missing page is brought back into the memory guest is notified and task resumes execution.

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/include/asm/kvm_para.h |    3 +
 arch/x86/kernel/kvm.c           |  132 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+), 0 deletions(-)

arch/x86/include/asm/kvm_para.h +3 -0

index d7d7079..79bb7f2 100644
@@ -49,6 +49,9 @@ struct kvm_mmu_op_release_pt {
 	__u64 pt_phys;
 };
 
+#define KVM_PV_REASON_PAGE_NOT_PRESENT 1
+#define KVM_PV_REASON_PAGE_READY 2
+
 struct kvm_vcpu_pv_apf_data {
 	__u32 reason;
 	__u32 enabled;

arch/x86/kernel/kvm.c +132 -0

161 lines not on this page · raw message

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Avi Kivity

2009-11-25 12:45 UTC #8 lore raw

Re: [PATCH v2 05/12] Handle asynchronous page fault in a PV guest.

Quoted 23 lines · Gleb Natapov

Asynchronous page fault notifies vcpu that page it is trying to access is swapped out by a host. In response guest puts a task that caused the fault to sleep until page is swapped in again. When missing page is brought back into the memory guest is notified and task resumes execution.
arch/x86/kernel/kvm.c +17 -0
+
+static void apf_task_wait(struct task_struct *tsk, u32 token)
+{
+	u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
+	struct kvm_task_sleep_head *b =&async_pf_sleepers[key];
+	struct kvm_task_sleep_node n, *e;
+	DEFINE_WAIT(wait);
+
+	spin_lock(&b->lock);
+	e = _find_apf_task(b, token);
+	if (e) {
+		/* dummy entry exist ->  wake up was delivered ahead of PF */
+		hlist_del(&e->link);
+		kfree(e);
+		spin_unlock(&b->lock);
+		return;
+	}

arch/x86/kernel/kvm.c static void kvm_io_delay(void) · line 113

Quoted 10 lines

arch/x86/kernel/kvm.c +10 -0

+
+	n.token = token;
+	init_waitqueue_head(&n.wq);
+	hlist_add_head(&n.link,&b->list);
+	spin_unlock(&b->lock);
+
+	for (;;) {
+		prepare_to_wait(&n.wq,&wait, TASK_UNINTERRUPTIBLE);
+		if (hlist_unhashed(&n.link))
+			break;

This looks safe without b->lock, but please add a comment explaining why it is safe.

arch/x86/kernel/kvm.c static void kvm_io_delay(void) · line 164

Quoted 10 lines

arch/x86/kernel/kvm.c +9 -0

+int kvm_handle_pf(struct pt_regs *regs, unsigned long error_code)
+{
+	u32 reason, token;
+
+	if (!per_cpu(apf_reason, smp_processor_id()).enabled)
+		return 0;
+
+	reason = per_cpu(apf_reason, smp_processor_id()).reason;
+	per_cpu(apf_reason, smp_processor_id()).reason = 0;

Use __get_cpu_var(), shorter.

arch/x86/kernel/kvm.c static void __init kvm_smp_prepare_boot_cpu(void) · line 403

Quoted 6 lines

arch/x86/kernel/kvm.c +1 -0

@@ -270,11 +399,14 @@ static void __init kvm_smp_prepare_boot_cpu(void)
 
  void __init kvm_guest_init(void)
  {
+	int i;

Quoted 11 lines

arch/x86/kernel/kvm.c +2 -0

  	if (!kvm_para_available())
  		return;
 
  	paravirt_ops_setup();
  	register_reboot_notifier(&kvm_pv_reboot_nb);
+	for (i = 0; i<  KVM_TASK_SLEEP_HASHSIZE; i++)
+		spin_lock_init(&async_pf_sleepers[i].lock);
  #ifdef CONFIG_SMP
  	smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu;

  #else

Signature · 7 lines

error compiling committee.c: too many arguments to function

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:06 UTC #9 lore raw

PATCH v2 6/12 Export __get_user_pages_fast.

In reply to Gleb Natapov

KVM will use it to try and find a page without falling back to slow gup. That is why get_user_pages_fast() is not enough.

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/mm/gup.c |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

arch/x86/mm/gup.c +2 -0

index 71da1bc..cea0dfe 100644
@@ -8,6 +8,7 @@
 #include <linux/mm.h>
 #include <linux/vmstat.h>
 #include <linux/highmem.h>
+#include <linux/module.h>
 
 #include <asm/pgtable.h>

@@ -274,6 +275,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
 
 	return nr;
 }
+EXPORT_SYMBOL_GPL(__get_user_pages_fast);
 
 /**
  * get_user_pages_fast() - pin user pages in memory

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:06 UTC #10 lore raw

PATCH v2 7/12 Add get_user_pages() variant that fails if major fault is required.

In reply to Gleb Natapov

This patch add get_user_pages() variant that only succeeds if getting a reference to a page doesn't require major fault.

Reviewed-by Rik van Riel <[email protected]>

Signed-off-by Gleb Natapov <[email protected]>

---
 fs/ncpfs/mmap.c    |    2 ++
 include/linux/mm.h |    5 +++++
 mm/filemap.c       |    3 +++
 mm/memory.c        |   31 ++++++++++++++++++++++++++++---
 mm/shmem.c         |    8 +++++++-
 5 files changed, 45 insertions(+), 4 deletions(-)

fs/ncpfs/mmap.c +2 -0

index 15458de..338527e 100644
@@ -39,6 +39,8 @@ static int ncp_file_mmap_fault(struct vm_area_struct *area,
 	int bufsize;
 	int pos; /* XXX: loff_t ? */
 
+	if (vmf->flags & FAULT_FLAG_MINOR)
+		return VM_FAULT_MAJOR | VM_FAULT_ERROR;
 	/*
 	 * ncpfs has nothing against high pages as long
 	 * as recvmsg and memset works on it

include/linux/mm.h +5 -0

index 24c3956..2304181 100644
@@ -136,6 +136,7 @@ extern pgprot_t protection_map[16];
 #define FAULT_FLAG_WRITE	0x01	/* Fault was a write access */
 #define FAULT_FLAG_NONLINEAR	0x02	/* Fault was via a nonlinear mapping */
 #define FAULT_FLAG_MKWRITE	0x04	/* Fault was mkwrite of existing pte */
+#define FAULT_FLAG_MINOR	0x08	/* Do only minor fault */
 
 /*
  * This interface is used by x86 PAT code to identify a pfn mapping that is

@@ -821,6 +822,9 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *
 int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
 			unsigned long start, int nr_pages, int write, int force,
 			struct page **pages, struct vm_area_struct **vmas);
+int get_user_pages_noio(struct task_struct *tsk, struct mm_struct *mm,
+			unsigned long start, int nr_pages, int write, int force,
+			struct page **pages, struct vm_area_struct **vmas);
 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
 			struct page **pages);
 struct page *get_dump_page(unsigned long addr);

@@ -1239,6 +1243,7 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address,
 #define FOLL_GET	0x04	/* do get_page on page */
 #define FOLL_DUMP	0x08	/* give error on hole if it would be zero */
 #define FOLL_FORCE	0x10	/* get_user_pages read/write w/o permission */
+#define FOLL_MINOR	0x20	/* do only minor page faults */
 
 typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
 			void *data);

mm/filemap.c +3 -0

index ef169f3..6ef29e0 100644
@@ -1530,6 +1530,9 @@ int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 			goto no_cached_page;
 		}
 	} else {
+		if (vmf->flags & FAULT_FLAG_MINOR)
+			return VM_FAULT_MAJOR | VM_FAULT_ERROR;
+
 		/* No page in the page cache at all */
 		do_sync_mmap_readahead(vma, ra, file, offset);
 		count_vm_event(PGMAJFAULT);

mm/memory.c +28 -3

index 6ab19dd..f4da763 100644
@@ -1321,10 +1321,13 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
 			cond_resched();
 			while (!(page = follow_page(vma, start, foll_flags))) {
 				int ret;
+				unsigned int fault_fl =
+					((foll_flags & FOLL_WRITE) ?
+					FAULT_FLAG_WRITE : 0) |
+					((foll_flags & FOLL_MINOR) ?
+					FAULT_FLAG_MINOR : 0);
 
-				ret = handle_mm_fault(mm, vma, start,
-					(foll_flags & FOLL_WRITE) ?
-					FAULT_FLAG_WRITE : 0);
+				ret = handle_mm_fault(mm, vma, start, fault_fl);
 
 				if (ret & VM_FAULT_ERROR) {
 					if (ret & VM_FAULT_OOM)

@@ -1332,6 +1335,8 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
 					if (ret &
 					    (VM_FAULT_HWPOISON|VM_FAULT_SIGBUS))
 						return i ? i : -EFAULT;
+					else if (ret & VM_FAULT_MAJOR)
+						return i ? i : -EFAULT;
 					BUG();
 				}
 				if (ret & VM_FAULT_MAJOR)

@@ -1442,6 +1447,23 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
 }
 EXPORT_SYMBOL(get_user_pages);
 
+int get_user_pages_noio(struct task_struct *tsk, struct mm_struct *mm,
+		unsigned long start, int nr_pages, int write, int force,
+		struct page **pages, struct vm_area_struct **vmas)
+{
+	int flags = FOLL_TOUCH | FOLL_MINOR;
+
+	if (pages)
+		flags |= FOLL_GET;
+	if (write)
+		flags |= FOLL_WRITE;
+	if (force)
+		flags |= FOLL_FORCE;
+
+	return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
+}
+EXPORT_SYMBOL(get_user_pages_noio);
+
 /**
  * get_dump_page() - pin user page in memory while writing it to core dump
  * @addr: user address

@@ -2521,6 +2543,9 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
 	delayacct_set_flag(DELAYACCT_PF_SWAPIN);
 	page = lookup_swap_cache(entry);
 	if (!page) {
+		if (flags & FAULT_FLAG_MINOR)
+			return VM_FAULT_MAJOR | VM_FAULT_ERROR;
+
 		grab_swap_token(mm); /* Contend for token _before_ read-in */
 		page = swapin_readahead(entry,
 					GFP_HIGHUSER_MOVABLE, vma, address);

mm/shmem.c +7 -1

index 356dd99..6a9d3c0 100644
@@ -1218,6 +1218,7 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
 	swp_entry_t swap;
 	gfp_t gfp;
 	int error;
+	int flags = type ? *type : 0;
 
 	if (idx >= SHMEM_MAX_INDEX)
 		return -EFBIG;

@@ -1266,6 +1267,11 @@ repeat:
 		swappage = lookup_swap_cache(swap);
 		if (!swappage) {
 			shmem_swp_unmap(entry);
+			if (flags & FAULT_FLAG_MINOR) {
+				spin_unlock(&info->lock);
+				*type = VM_FAULT_MAJOR | VM_FAULT_ERROR;
+				goto failed;
+			}
 			/* here we actually do the io */
 			if (type && !(*type & VM_FAULT_MAJOR)) {
 				__count_vm_event(PGMAJFAULT);

@@ -1474,7 +1480,7 @@ static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
 	int error;
-	int ret;
+	int ret = (int)vmf->flags;
 
 	if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
 		return VM_FAULT_SIGBUS;

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:06 UTC #11 lore raw

PATCH v2 8/12 Inject asynchronous page fault into a guest if page is swapped out.

In reply to Gleb Natapov

If guest access swapped out memory do not swap it in from vcpu thread context. Setup slow work to do swapping and send async page fault to a guest.

Allow async page fault injection only when guest is in user mode since otherwise guest may be in non-sleepable context and will not be able to reschedule.

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |   13 +++
 arch/x86/kvm/Kconfig            |    2 +
 arch/x86/kvm/mmu.c              |   36 ++++++-
 arch/x86/kvm/paging_tmpl.h      |   16 +++-
 arch/x86/kvm/x86.c              |   37 ++++++-
 include/linux/kvm_host.h        |   27 +++++
 include/trace/events/kvm.h      |   60 +++++++++++
 virt/kvm/Kconfig                |    3 +
 virt/kvm/kvm_main.c             |  225 ++++++++++++++++++++++++++++++++++++++-
 9 files changed, 412 insertions(+), 7 deletions(-)

arch/x86/include/asm/kvm_host.h +13 -0

index 9598e85..ad177a4 100644
@@ -377,6 +377,7 @@ struct kvm_vcpu_arch {
 
 	u32 __user *apf_data;
 	u64 apf_msr_val;
+	u32 async_pf_id;
 };
 
 struct kvm_mem_alias {

@@ -539,6 +540,10 @@ struct kvm_x86_ops {
 	const struct trace_print_flags *exit_reasons_str;
 };
 
+struct kvm_arch_async_pf {
+	u32 token;
+};
+
 extern struct kvm_x86_ops *kvm_x86_ops;
 
 int kvm_mmu_module_init(void);

@@ -817,4 +822,12 @@ int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
 void kvm_define_shared_msr(unsigned index, u32 msr);
 void kvm_set_shared_msr(unsigned index, u64 val);
 
+struct kvm_async_pf;
+
+void kvm_arch_inject_async_page_not_present(struct kvm_vcpu *vcpu,
+					    struct kvm_async_pf *work);
+void kvm_arch_inject_async_page_present(struct kvm_vcpu *vcpu,
+					struct kvm_async_pf *work);
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
 #endif /* _ASM_X86_KVM_HOST_H */
+

arch/x86/kvm/Kconfig +2 -0

index 4cd4983..f77b127 100644
@@ -28,6 +28,8 @@ config KVM
 	select HAVE_KVM_IRQCHIP
 	select HAVE_KVM_EVENTFD
 	select KVM_APIC_ARCHITECTURE
+	select KVM_ASYNC_PF
+	select SLOW_WORK
 	select USER_RETURN_NOTIFIER
 	---help---
 	  Support hosting fully virtualized guest machines using hardware

arch/x86/kvm/mmu.c +35 -1

index 4c3e5b2..2cdf3e3 100644
@@ -19,6 +19,7 @@
 
 #include "mmu.h"
 #include "kvm_cache_regs.h"
+#include "x86.h"
 
 #include <linux/kvm_host.h>
 #include <linux/types.h>

@@ -29,6 +30,8 @@
 #include <linux/swap.h>
 #include <linux/hugetlb.h>
 #include <linux/compiler.h>
+#include <trace/events/kvm.h>
+#undef TRACE_INCLUDE_FILE
 
 #include <asm/page.h>
 #include <asm/cmpxchg.h>

@@ -2189,6 +2192,21 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
 			     error_code & PFERR_WRITE_MASK, gfn);
 }
 
+int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
+{
+	struct kvm_arch_async_pf arch;
+	arch.token = (vcpu->arch.async_pf_id++ << 12) | vcpu->vcpu_id;
+	return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
+}
+
+static bool can_do_async_pf(struct kvm_vcpu *vcpu)
+{
+	if (!vcpu->arch.apf_data || kvm_event_needs_reinjection(vcpu))
+		return false;
+
+	return !!kvm_x86_ops->get_cpl(vcpu);
+}
+
 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
 				u32 error_code)
 {

@@ -2211,7 +2229,23 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
 
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
-	pfn = gfn_to_pfn(vcpu->kvm, gfn);
+
+	if (can_do_async_pf(vcpu)) {
+		r = gfn_to_pfn_async(vcpu->kvm, gfn, &pfn);
+		trace_kvm_try_async_get_page(r, pfn);
+	} else {
+do_sync:
+		r = 1;
+		pfn = gfn_to_pfn(vcpu->kvm, gfn);
+	}
+
+	if (!r) {
+		if (!kvm_arch_setup_async_pf(vcpu, gpa, gfn))
+			goto do_sync;
+		return 0;
+	}
+
+	/* mmio */
 	if (is_error_pfn(pfn)) {
 		kvm_release_pfn_clean(pfn);
 		return 1;

arch/x86/kvm/paging_tmpl.h +15 -1

index a601713..44d19dc 100644
@@ -419,7 +419,21 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
-	pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
+
+	if (can_do_async_pf(vcpu)) {
+		r = gfn_to_pfn_async(vcpu->kvm, walker.gfn, &pfn);
+		trace_kvm_try_async_get_page(r, pfn);
+	} else {
+do_sync:
+		r = 1;
+		pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
+	}
+
+	if (!r) {
+		if (!kvm_arch_setup_async_pf(vcpu, addr, walker.gfn))
+			goto do_sync;
+		return 0;
+	}
 
 	/* mmio */
 	if (is_error_pfn(pfn)) {

arch/x86/kvm/x86.c +33 -4

index ce8e66d..cbbe5fd 100644
@@ -3898,6 +3898,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		}
 	}
 
+	kvm_check_async_pf_completion(vcpu);
+
 	preempt_disable();
 
 	kvm_x86_ops->prepare_guest_switch(vcpu);

@@ -5151,8 +5153,10 @@ static void kvm_free_vcpus(struct kvm *kvm)
 	/*
 	 * Unpin any mmu pages first.
 	 */
-	kvm_for_each_vcpu(i, vcpu, kvm)
+	kvm_for_each_vcpu(i, vcpu, kvm) {
+		kvm_clear_async_pf_completion_queue(vcpu);
 		kvm_unload_vcpu_mmu(vcpu);
+	}
 	kvm_for_each_vcpu(i, vcpu, kvm)
 		kvm_arch_vcpu_free(vcpu);

@@ -5251,10 +5255,11 @@ void kvm_arch_flush_shadow(struct kvm *kvm)
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
+		|| !list_empty_careful(&vcpu->async_pf_done)
 		|| vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
-		|| vcpu->arch.nmi_pending ||
-		(kvm_arch_interrupt_allowed(vcpu) &&
-		 kvm_cpu_has_interrupt(vcpu));
+		|| vcpu->arch.nmi_pending
+		|| (kvm_arch_interrupt_allowed(vcpu) &&
+		    kvm_cpu_has_interrupt(vcpu));
 }
 
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)

@@ -5301,6 +5306,30 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 }
 EXPORT_SYMBOL_GPL(kvm_set_rflags);
 
+void kvm_arch_inject_async_page_not_present(struct kvm_vcpu *vcpu,
+					    struct kvm_async_pf *work)
+{
+	put_user(KVM_PV_REASON_PAGE_NOT_PRESENT, vcpu->arch.apf_data);
+	kvm_inject_page_fault(vcpu, work->arch.token, 0);
+	trace_kvm_send_async_pf(work->arch.token, work->gva,
+				KVM_PV_REASON_PAGE_NOT_PRESENT);
+}
+
+void kvm_arch_inject_async_page_present(struct kvm_vcpu *vcpu,
+					struct kvm_async_pf *work)
+{
+	put_user(KVM_PV_REASON_PAGE_READY, vcpu->arch.apf_data);
+	kvm_inject_page_fault(vcpu, work->arch.token, 0);
+	trace_kvm_send_async_pf(work->arch.token, work->gva,
+				KVM_PV_REASON_PAGE_READY);
+}
+
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
+{
+	return !kvm_event_needs_reinjection(vcpu) &&
+		kvm_x86_ops->interrupt_allowed(vcpu);
+}
+
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);

include/linux/kvm_host.h +27 -0

index bd5a616..7579685 100644
@@ -16,6 +16,7 @@
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/msi.h>
+#include <linux/slow-work.h>
 #include <asm/signal.h>
 
 #include <linux/kvm.h>

@@ -72,6 +73,25 @@ void __kvm_io_bus_unregister_dev(struct kvm_io_bus *bus,
 void kvm_io_bus_unregister_dev(struct kvm *kvm, struct kvm_io_bus *bus,
 			       struct kvm_io_device *dev);
 
+#ifdef CONFIG_KVM_ASYNC_PF
+struct kvm_async_pf {
+	struct slow_work work;
+	struct list_head link;
+	struct kvm_vcpu *vcpu;
+	struct mm_struct *mm;
+	gva_t gva;
+	unsigned long addr;
+	struct kvm_arch_async_pf arch;
+	struct page *page;
+	atomic_t used;
+};
+
+void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
+void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
+int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn,
+		       struct kvm_arch_async_pf *arch);
+#endif
+
 struct kvm_vcpu {
 	struct kvm *kvm;
 #ifdef CONFIG_PREEMPT_NOTIFIERS

@@ -99,6 +119,12 @@ struct kvm_vcpu {
 	gpa_t mmio_phys_addr;
 #endif
 
+#ifdef CONFIG_KVM_ASYNC_PF
+	struct list_head async_pf_done;
+	spinlock_t async_pf_lock;
+	struct kvm_async_pf *async_pf_work;
+#endif
+
 	struct kvm_vcpu_arch arch;
 };

@@ -263,6 +289,7 @@ void kvm_release_page_dirty(struct page *page);
 void kvm_set_page_dirty(struct page *page);
 void kvm_set_page_accessed(struct page *page);
 
+int gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, pfn_t *pfn);
 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
 void kvm_release_pfn_dirty(pfn_t);
 void kvm_release_pfn_clean(pfn_t pfn);

include/trace/events/kvm.h +60 -0

66 lines not on this page · raw message

virt/kvm/Kconfig +3 -0

index daece36..ccbc47a 100644
@@ -12,3 +12,6 @@ config HAVE_KVM_EVENTFD
 
 config KVM_APIC_ARCHITECTURE
        bool
+
+config KVM_ASYNC_PF
+       bool

virt/kvm/kvm_main.c +224 -1

274 lines not on this page · raw message

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Avi Kivity

2009-11-25 13:03 UTC #12 lore raw

Re: [PATCH v2 08/12] Inject asynchronous page fault into a guest if page is swapped out.

Quoted 9 lines · Gleb Natapov

If guest access swapped out memory do not swap it in from vcpu thread context. Setup slow work to do swapping and send async page fault to a guest.
Allow async page fault injection only when guest is in user mode since otherwise guest may be in non-sleepable context and will not be able to reschedule.

arch/x86/kvm/x86.c void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) · line 5325

Quoted 9 lines

arch/x86/kvm/x86.c +9 -0

+
+void kvm_arch_inject_async_page_present(struct kvm_vcpu *vcpu,
+					struct kvm_async_pf *work)
+{
+	put_user(KVM_PV_REASON_PAGE_READY, vcpu->arch.apf_data);
+	kvm_inject_page_fault(vcpu, work->arch.token, 0);
+	trace_kvm_send_async_pf(work->arch.token, work->gva,
+				KVM_PV_REASON_PAGE_READY);
+}

What if the guest is now handling a previous asynv pf or ready notification? We're clobbering the data structure.

arch/x86/kvm/x86.c void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) · line 5331

Quoted 7 lines

arch/x86/kvm/x86.c +6 -0

+
+bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
+{
+	return !kvm_event_needs_reinjection(vcpu)&&
+		kvm_x86_ops->interrupt_allowed(vcpu);
+}

Okay, so this is only allowed with interrupts disabled. Need to make sure the entire pf path up to async pf executes with interrupts disabled.

Signature · 7 lines

error compiling committee.c: too many arguments to function

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:06 UTC #13 lore raw

PATCH v2 9/12 Retry fault before vmentry

In reply to Gleb Natapov

When page is swapped in it is mapped into guest memory only after guest tries to access it again and generate another fault. To save this fault we can map it immediately since we know that guest is going to access the page.

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/include/asm/kvm_host.h |    6 +++++-
 arch/x86/kvm/mmu.c              |   15 +++++++++------
 arch/x86/kvm/paging_tmpl.h      |   38 +++++++++++++++++++++++++++++++++++---
 arch/x86/kvm/x86.c              |    7 +++++++
 virt/kvm/kvm_main.c             |    2 ++
 5 files changed, 58 insertions(+), 10 deletions(-)

arch/x86/include/asm/kvm_host.h +5 -1

index ad177a4..39009a4 100644
@@ -254,7 +254,7 @@ struct kvm_pio_request {
  */
 struct kvm_mmu {
 	void (*new_cr3)(struct kvm_vcpu *vcpu);
-	int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err);
+	int (*page_fault)(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gva, u32 err);
 	void (*free)(struct kvm_vcpu *vcpu);
 	gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva);
 	void (*prefetch_page)(struct kvm_vcpu *vcpu,

@@ -542,6 +542,8 @@ struct kvm_x86_ops {
 
 struct kvm_arch_async_pf {
 	u32 token;
+	gpa_t cr3;
+	u32 error_code;
 };
 
 extern struct kvm_x86_ops *kvm_x86_ops;

@@ -828,6 +830,8 @@ void kvm_arch_inject_async_page_not_present(struct kvm_vcpu *vcpu,
 					    struct kvm_async_pf *work);
 void kvm_arch_inject_async_page_present(struct kvm_vcpu *vcpu,
 					struct kvm_async_pf *work);
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
+			       struct kvm_async_pf *work);
 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu);
 #endif /* _ASM_X86_KVM_HOST_H */

arch/x86/kvm/mmu.c +9 -6

index 2cdf3e3..1225c31 100644
@@ -2172,7 +2172,7 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
 	return vaddr;
 }
 
-static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
+static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gva,
 				u32 error_code)
 {
 	gfn_t gfn;

@@ -2192,10 +2192,13 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
 			     error_code & PFERR_WRITE_MASK, gfn);
 }
 
-int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
+int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gva,
+			    gfn_t gfn, u32 error_code)
 {
 	struct kvm_arch_async_pf arch;
 	arch.token = (vcpu->arch.async_pf_id++ << 12) | vcpu->vcpu_id;
+	arch.cr3 = cr3;
+	arch.error_code = error_code;
 	return kvm_setup_async_pf(vcpu, gva, gfn, &arch);
 }

@@ -2207,7 +2210,7 @@ static bool can_do_async_pf(struct kvm_vcpu *vcpu)
 	return !!kvm_x86_ops->get_cpl(vcpu);
 }
 
-static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
+static int tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gpa,
 				u32 error_code)
 {
 	pfn_t pfn;

@@ -2230,7 +2233,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 
-	if (can_do_async_pf(vcpu)) {
+	if (cr3 == vcpu->arch.cr3 && can_do_async_pf(vcpu)) {
 		r = gfn_to_pfn_async(vcpu->kvm, gfn, &pfn);
 		trace_kvm_try_async_get_page(r, pfn);
 	} else {

@@ -2240,7 +2243,7 @@ do_sync:
 	}
 
 	if (!r) {
-		if (!kvm_arch_setup_async_pf(vcpu, gpa, gfn))
+		if (!kvm_arch_setup_async_pf(vcpu, cr3, gpa, gfn, error_code))
 			goto do_sync;
 		return 0;
 	}

@@ -2810,7 +2813,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
 	int r;
 	enum emulation_result er;
 
-	r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
+	r = vcpu->arch.mmu.page_fault(vcpu, vcpu->arch.cr3, cr2, error_code);
 	if (r < 0)
 		goto out;

arch/x86/kvm/paging_tmpl.h +35 -3

index 44d19dc..702893c 100644
@@ -375,7 +375,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
  *  Returns: 1 if we need to emulate the instruction, 0 otherwise, or
  *           a negative value on error.
  */
-static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
+static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t addr,
 			       u32 error_code)
 {
 	int write_fault = error_code & PFERR_WRITE_MASK;

@@ -388,6 +388,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 	pfn_t pfn;
 	int level = PT_PAGE_TABLE_LEVEL;
 	unsigned long mmu_seq;
+	gpa_t curr_cr3 = vcpu->arch.cr3;
 
 	pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
 	kvm_mmu_audit(vcpu, "pre page fault");

@@ -396,6 +397,19 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 	if (r)
 		return r;
 
+	if (curr_cr3 != cr3) {
+		/*
+		 * We do page fault on behaltf of a process that is sleeping
+		 * because of async PF. PV guest shouldn't kill process while
+		 * it waits for host to swap-in the page so cr3 has to be
+		 * valid here.
+		 */
+		vcpu->arch.cr3 = cr3;
+		paging_new_cr3(vcpu);
+		if (kvm_mmu_reload(vcpu))
+			goto switch_cr3;
+	}
+
 	/*
 	 * Look up the guest pte for the faulting address.
 	 */

@@ -406,6 +420,8 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 	 * The page is not mapped by the guest.  Let the guest handle it.
 	 */
 	if (!r) {
+		if (curr_cr3 != vcpu->arch.cr3)
+			goto switch_cr3;
 		pgprintk("%s: guest page fault\n", __func__);
 		inject_page_fault(vcpu, addr, walker.error_code);
 		vcpu->arch.last_pt_write_count = 0; /* reset fork detector */

@@ -420,7 +436,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 
-	if (can_do_async_pf(vcpu)) {
+	if (curr_cr3 == vcpu->arch.cr3 && can_do_async_pf(vcpu)) {
 		r = gfn_to_pfn_async(vcpu->kvm, walker.gfn, &pfn);
 		trace_kvm_try_async_get_page(r, pfn);
 	} else {

@@ -430,13 +446,18 @@ do_sync:
 	}
 
 	if (!r) {
-		if (!kvm_arch_setup_async_pf(vcpu, addr, walker.gfn))
+		if (!kvm_arch_setup_async_pf(vcpu, cr3, addr, walker.gfn,
+					     error_code))
 			goto do_sync;
+		if (curr_cr3 != vcpu->arch.cr3)
+			goto switch_cr3;
 		return 0;
 	}
 
 	/* mmio */
 	if (is_error_pfn(pfn)) {
+		if (curr_cr3 != vcpu->arch.cr3)
+			goto switch_cr3;
 		pgprintk("gfn %lx is mmio\n", walker.gfn);
 		kvm_release_pfn_clean(pfn);
 		return 1;

@@ -458,12 +479,23 @@ do_sync:
 	kvm_mmu_audit(vcpu, "post page fault (fixed)");
 	spin_unlock(&vcpu->kvm->mmu_lock);
 
+	if (curr_cr3 != vcpu->arch.cr3)
+		goto switch_cr3;
+
 	return write_pt;
 
 out_unlock:
 	spin_unlock(&vcpu->kvm->mmu_lock);
 	kvm_release_pfn_clean(pfn);
 	return 0;
+switch_cr3:
+	if (curr_cr3 != vcpu->arch.cr3) {
+		vcpu->arch.cr3 = curr_cr3;
+		paging_new_cr3(vcpu);
+		kvm_mmu_reload(vcpu);
+	}
+
+	return write_pt;
 }
 
 static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)

arch/x86/kvm/x86.c +7 -0

index cbbe5fd..c29af1d 100644
@@ -5306,6 +5306,13 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 }
 EXPORT_SYMBOL_GPL(kvm_set_rflags);
 
+void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
+			       struct kvm_async_pf *work)
+{
+	vcpu->arch.mmu.page_fault(vcpu, work->arch.cr3, work->gva,
+				  work->arch.error_code);
+}
+
 void kvm_arch_inject_async_page_not_present(struct kvm_vcpu *vcpu,
 					    struct kvm_async_pf *work)
 {

virt/kvm/kvm_main.c +2 -0

index 14ac02a..6e6769f 100644
@@ -1212,6 +1212,7 @@ void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu)
 			spin_lock(&vcpu->async_pf_lock);
 			list_del(&work->link);
 			spin_unlock(&vcpu->async_pf_lock);
+			kvm_arch_async_page_ready(vcpu, work);
 			put_page(work->page);
 			async_pf_work_free(work);
 		}

@@ -1226,6 +1227,7 @@ void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu)
 	list_del(&work->link);
 	spin_unlock(&vcpu->async_pf_lock);
 
+	kvm_arch_async_page_ready(vcpu, work);
 	kvm_arch_inject_async_page_present(vcpu, work);
 
 	put_page(work->page);

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Avi Kivity

2009-11-25 13:09 UTC #14 lore raw

Re: [PATCH v2 09/12] Retry fault before vmentry

Quoted 12 lines · Gleb Natapov

When page is swapped in it is mapped into guest memory only after guest tries to access it again and generate another fault. To save this fault we can map it immediately since we know that guest is going to access the page.
arch/x86/kvm/mmu.c +1 -1
-static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
+static int tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gpa,
  				u32 error_code)
  {
  	pfn_t pfn;

arch/x86/kvm/mmu.c static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, · line 2237

Quoted 6 lines

arch/x86/kvm/mmu.c +1 -1

@@ -2230,7 +2233,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
  	mmu_seq = vcpu->kvm->mmu_notifier_seq;
  	smp_rmb();
 
-	if (can_do_async_pf(vcpu)) {
+	if (cr3 == vcpu->arch.cr3&&  can_do_async_pf(vcpu)) {

Why check cr3 here?

arch/x86/kvm/paging_tmpl.h static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, · line 379

Quoted 4 lines

arch/x86/kvm/paging_tmpl.h +1 -1

-static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
+static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t addr,
  			       u32 error_code)

I'd be slightly happier if we had a page_fault_other_cr3() op that switched cr3, called the original, then switched back (the tdp version need not change anything).

Signature · 7 lines

error compiling committee.c: too many arguments to function

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-25 13:20 UTC #15 lore raw

Re: [PATCH v2 09/12] Retry fault before vmentry

Quoted 16 lines · Avi Kivity

Quoted 15 lines · Gleb Natapov
When page is swapped in it is mapped into guest memory only after guest tries to access it again and generate another fault. To save this fault we can map it immediately since we know that guest is going to access the page.
arch/x86/kvm/mmu.c +1 -1
-static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
+static int tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gpa,
 				u32 error_code)
 {
 	pfn_t pfn;
@@ -2230,7 +2233,7 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 

Quoted 5 lines

Review static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa, · line 2236
Quoted 2 lines
(quoted diff) +1 -1
-	if (can_do_async_pf(vcpu)) {
+	if (cr3 == vcpu->arch.cr3&&  can_do_async_pf(vcpu)) {
Why check cr3 here?

If cr3 == vcpu->arch.cr3 here we know that this is guest generated page fault so we try to do it async. Otherwise this is async page fault code try to establish mapping, so need to go through async logic. Theoretically page that was just swapped in can be swapped out once again at this point and in this case we need to go to sleep here otherwise things may go wrong.

Quoted 10 lines

arch/x86/kvm/paging_tmpl.h static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, · line 379
Quoted 3 lines
arch/x86/kvm/paging_tmpl.h +1 -1
-static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
+static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t addr,
 			       u32 error_code)
I'd be slightly happier if we had a page_fault_other_cr3() op that switched cr3, called the original, then switched back (the tdp version need not change anything).
Signature · 1 line
error compiling committee.c: too many arguments to function

--
Gleb.

Gleb Natapov

author

2009-11-23 14:06 UTC #16 lore raw

PATCH v2 10/12 Maintain preemptability count even for !CONFIG_PREEMPT kernels

In reply to Gleb Natapov

Do not preempt kernel. Just maintain counter to know if task can be rescheduled.
Asynchronous page fault may be delivered while spinlock is held or current
process can't be preempted for other reasons. KVM uses preempt_count() to check if preemptions is allowed and schedule other process if possible. This works with preemptable kernels since they maintain accurate information about
preemptability in preempt_count. This patch make non-preemptable kernel
maintain accurate information in preempt_count too.

Signed-off-by Gleb Natapov <[email protected]>

---
 include/linux/hardirq.h |   14 +++-----------
 include/linux/preempt.h |   22 ++++++++++++++++------
 include/linux/sched.h   |    4 ----
 kernel/sched.c          |    6 ------
 lib/kernel_lock.c       |    1 +
 5 files changed, 20 insertions(+), 27 deletions(-)

include/linux/hardirq.h +3 ~~-11~~

index 6d527ee..484ba38 100644
@@ -2,9 +2,7 @@
 #define LINUX_HARDIRQ_H
 
 #include <linux/preempt.h>
-#ifdef CONFIG_PREEMPT
 #include <linux/smp_lock.h>
-#endif
 #include <linux/lockdep.h>
 #include <linux/ftrace_irq.h>
 #include <asm/hardirq.h>

@@ -92,13 +90,8 @@
  */
 #define in_nmi()	(preempt_count() & NMI_MASK)
 
-#if defined(CONFIG_PREEMPT)
-# define PREEMPT_INATOMIC_BASE kernel_locked()
-# define PREEMPT_CHECK_OFFSET 1
-#else
-# define PREEMPT_INATOMIC_BASE 0
-# define PREEMPT_CHECK_OFFSET 0
-#endif
+#define PREEMPT_CHECK_OFFSET 1
+#define PREEMPT_INATOMIC_BASE kernel_locked()
 
 /*
  * Are we running in atomic context?  WARNING: this macro cannot

@@ -116,12 +109,11 @@
 #define in_atomic_preempt_off() \
 		((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
 
+#define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1)
 #ifdef CONFIG_PREEMPT
 # define preemptible()	(preempt_count() == 0 && !irqs_disabled())
-# define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1)
 #else
 # define preemptible()	0
-# define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
 
 #if defined(CONFIG_SMP) || defined(CONFIG_GENERIC_HARDIRQS)

include/linux/preempt.h +16 -6

index 72b1a10..7d039ca 100644
@@ -82,14 +82,24 @@ do { \
 
 #else
 
-#define preempt_disable()		do { } while (0)
-#define preempt_enable_no_resched()	do { } while (0)
-#define preempt_enable()		do { } while (0)
+#define preempt_disable() \
+do { \
+	inc_preempt_count(); \
+	barrier(); \
+} while (0)
+
+#define preempt_enable() \
+do { \
+	barrier(); \
+	dec_preempt_count(); \
+} while (0)
+
+#define preempt_enable_no_resched()	preempt_enable()
 #define preempt_check_resched()		do { } while (0)
 
-#define preempt_disable_notrace()		do { } while (0)
-#define preempt_enable_no_resched_notrace()	do { } while (0)
-#define preempt_enable_notrace()		do { } while (0)
+#define preempt_disable_notrace()		preempt_disable()
+#define preempt_enable_no_resched_notrace()	preempt_enable()
+#define preempt_enable_notrace()		preempt_enable()
 
 #endif

include/linux/sched.h +0 -4

index 75e6e60..1895486 100644
@@ -2379,11 +2379,7 @@ extern int _cond_resched(void);
 
 extern int __cond_resched_lock(spinlock_t *lock);
 
-#ifdef CONFIG_PREEMPT
 #define PREEMPT_LOCK_OFFSET	PREEMPT_OFFSET
-#else
-#define PREEMPT_LOCK_OFFSET	0
-#endif
 
 #define cond_resched_lock(lock) ({				\
 	__might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);	\

kernel/sched.c +0 -6

index 3c11ae0..92ce282 100644
@@ -2590,10 +2590,8 @@ void sched_fork(struct task_struct *p, int clone_flags)
 #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
 	p->oncpu = 0;
 #endif
-#ifdef CONFIG_PREEMPT
 	/* Want to start with kernel preemption disabled. */
 	task_thread_info(p)->preempt_count = 1;
-#endif
 	plist_node_init(&p->pushable_tasks, MAX_PRIO);
 
 	put_cpu();

@@ -6973,11 +6971,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
 	spin_unlock_irqrestore(&rq->lock, flags);
 
 	/* Set the preempt count _outside_ the spinlocks! */
-#if defined(CONFIG_PREEMPT)
 	task_thread_info(idle)->preempt_count = (idle->lock_depth >= 0);
-#else
-	task_thread_info(idle)->preempt_count = 0;
-#endif
 	/*
 	 * The idle tasks have their own, simple scheduling class:
 	 */

lib/kernel_lock.c +1 -0

index 39f1029..6e2659d 100644
@@ -93,6 +93,7 @@ static inline void __lock_kernel(void)
  */
 static inline void __lock_kernel(void)
 {
+	preempt_disable();
 	_raw_spin_lock(&kernel_flag);
 }
 #endif

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Peter Zijlstra

maintainer

2009-11-23 15:34 UTC #17 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 6 lines · Gleb Natapov

Do not preempt kernel. Just maintain counter to know if task can be rescheduled.
Asynchronous page fault may be delivered while spinlock is held or current
process can't be preempted for other reasons. KVM uses preempt_count() to check if preemptions is allowed and schedule other process if possible. This works with preemptable kernels since they maintain accurate information about
preemptability in preempt_count. This patch make non-preemptable kernel
maintain accurate information in preempt_count too.

I'm thinking you're going to have to convince some people this won't slow them down for no good.

Personally I always have PREEMPT=y, but other people seem to feel strongly about not doing so.

Gleb Natapov

author

2009-11-23 15:58 UTC #18 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 11 lines · Peter Zijlstra

Quoted 6 lines · Gleb Natapov
Do not preempt kernel. Just maintain counter to know if task can be rescheduled.
Asynchronous page fault may be delivered while spinlock is held or current
process can't be preempted for other reasons. KVM uses preempt_count() to check if preemptions is allowed and schedule other process if possible. This works with preemptable kernels since they maintain accurate information about
preemptability in preempt_count. This patch make non-preemptable kernel
maintain accurate information in preempt_count too.
I'm thinking you're going to have to convince some people this won't slow them down for no good.

I saw old discussions about this in mailing list archives. Usually someone wanted to use in_atomic() in driver code and this, of course, caused the resistant. In this case, I think, the use is legitimate.

Quoted 3 lines

Personally I always have PREEMPT=y, but other people seem to feel strongly about not doing so.

It is possible to add one more config option to enable reliable preempt_count() without enabling preemption or make async pf be dependable on PREEMPT=y. Don't like both of this options especially first one. There are more then enough options already.

--
Gleb.

Christoph Lameter

2009-11-23 17:30 UTC #19 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

This adds significant overhead for the !PREEMPT case adding lots of code in critical paths all over the place.

Gleb Natapov

author

2009-11-24 07:12 UTC #20 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 4 lines · Christoph Lameter

This adds significant overhead for the !PREEMPT case adding lots of code in critical paths all over the place.

I want to measure it. Can you suggest benchmarks to try?

--
Gleb.

Christoph Lameter

2009-11-24 15:14 UTC #21 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 4 lines · Gleb Natapov

Quoted 2 lines · Christoph Lameter
This adds significant overhead for the !PREEMPT case adding lots of code in critical paths all over the place.
I want to measure it. Can you suggest benchmarks to try?

AIM9 (reaim9)?

Any test suite will do that tests OS performance.

Latency will also be negatively impacted. There are already significant regressions in recent kernel releases so many of us who are sensitive to these issues just stick with old kernels (2.6.22 f.e.) and hope that the upstream issues are worked out at some point.

There is also lldiag package in my directory. See

http://www.kernel.org/pub/linux/kernel/people/christoph/lldiag

Try the latency test and the mcast test. Localhost multicast is typically a good test for kernel performance.

There is also the page fault test that Kamezawa-san posted recently in the thread where we tried to deal with the long term mmap_sem issues.

Gleb Natapov

author

2009-11-30 10:56 UTC #22 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 8 lines · Christoph Lameter

Quoted 4 lines · Gleb Natapov
Quoted 2 lines · Christoph Lameter
This adds significant overhead for the !PREEMPT case adding lots of code in critical paths all over the place.
I want to measure it. Can you suggest benchmarks to try?
AIM9 (reaim9)?

Below are results for kernel 2.6.32-rc8 with and without the patch (only this single patch is applied).

test name with (stddev) without (stddev)
===========================================================================

jmp_test         57853.762 ( 1086.51)  55664.287 ( 5152.14)    3.93%
stream_pipe      10286.967 (  132.01)  11396.327 (  306.01)   -9.73%
new_raph         12573.395 (    2.64)  12535.764 (   85.14)    0.30%
sync_disk_rw         0.100 (    0.00)      0.100 (    0.00)   -0.44%
udp_test          4008.058 (   37.57)   3774.514 (   22.03)    6.19%
add_long            68.542 (    0.00)     68.530 (    0.01)    0.02%
exec_test          181.615 (    0.46)    184.503 (    0.42)   -1.57%
div_double         114.209 (    0.02)    114.230 (    0.03)   -0.02%
mem_rtns_1         283.733 (    3.27)    285.936 (    2.24)   -0.77%
sync_disk_cp         0.043 (    0.00)      0.043 (    0.00)    0.03%
fun_cal2           780.701 (    0.16)    780.867 (    0.07)   -0.02%
matrix_rtns      70160.568 (   28.58)  70181.900 (   16.46)   -0.03%
fun_cal1           780.701 (    0.16)    780.763 (    0.13)   -0.01%
div_int            219.216 (    0.03)    219.264 (    0.04)   -0.02%
pipe_cpy         16239.120 (  468.99)  16727.067 (  280.27)   -2.92%
fifo_test        12864.276 (  242.82)  13383.616 (  199.31)   -3.88%
sync_disk_wrt        0.043 (    0.00)      0.043 (    0.00)   -0.11%
mul_long          4276.703 (    0.79)   4277.528 (    0.65)   -0.02%
num_rtns_1        4308.165 (    5.99)   4306.133 (    5.84)    0.05%
disk_src          1507.993 (    8.04)   1586.100 (    5.44)   -4.92%
mul_short         3422.840 (    0.31)   3423.280 (    0.24)   -0.01%
series_1        121706.708 (  266.62) 121356.355 (  982.04)    0.29%
mul_int           4277.353 (    0.45)   4277.953 (    0.34)   -0.01%
mul_float           99.947 (    0.02)     99.947 (    0.02)   -0.00%
link_test         2319.090 (   12.51)   2466.564 (    1.52)   -5.98%
fun_cal15          380.836 (    0.06)    380.876 (    0.10)   -0.01%
trig_rtns          163.416 (    0.13)    163.185 (    0.51)    0.14%
fun_cal            915.226 (    4.56)    902.033 (    1.44)    1.46%
misc_rtns_1       4285.322 (   18.72)   4282.907 (   27.07)    0.06%
brk_test           221.167 (    8.98)    230.345 (    7.98)   -3.98%
add_float          133.242 (    0.02)    133.249 (    0.02)   -0.01%
page_test          284.488 (    3.71)    284.180 (   13.91)    0.11%
div_long            85.364 (    0.27)     85.222 (    0.02)    0.17%
dir_rtns_1         207.953 (    2.56)    212.532 (    0.59)   -2.15%
disk_cp             66.449 (    0.43)     65.754 (    0.61)    1.06%
sieve               23.538 (    0.01)     23.599 (    0.11)   -0.26%
tcp_test          2085.428 (   18.43)   2059.062 (    5.52)    1.28%
disk_wrt            81.839 (    0.16)     82.652 (    0.41)   -0.98%
mul_double          79.951 (    0.01)     79.961 (    0.02)   -0.01%
fork_test           57.408 (    0.43)     57.835 (    0.27)   -0.74%
add_short          171.326 (    0.03)    171.314 (    0.01)    0.01%
creat-clo          395.995 (    3.63)    403.918 (    2.74)   -1.96%
sort_rtns_1        276.833 (   31.80)    290.855 (    0.46)   -4.82%
add_int             79.961 (    0.02)     79.967 (    0.00)   -0.01%
disk_rr             67.635 (    0.23)     68.282 (    0.59)   -0.95%
div_short          210.318 (    0.04)    210.365 (    0.05)   -0.02%
disk_rw             57.041 (    0.26)     57.470 (    0.31)   -0.75%
dgram_pipe       10088.191 (   86.81)   9848.119 (  406.33)    2.44%
shell_rtns_3       681.882 (    3.30)    693.734 (    2.67)   -1.71%
shell_rtns_2       681.721 (    3.24)    693.307 (    2.90)   -1.67%
shell_rtns_1       681.116 (    3.46)    692.302 (    3.16)   -1.62%
div_float          114.224 (    0.02)    114.230 (    0.00)   -0.01%
ram_copy        217812.436 (  615.62) 218160.548 (  135.66)   -0.16%
shared_memory    11022.611 (   20.75)  10870.031 (   61.44)    1.40%
signal_test        700.907 (    1.42)    711.253 (    0.49)   -1.46%
add_double          88.836 (    0.00)     88.837 (    0.00)   -0.00%
array_rtns         119.369 (    0.06)    119.182 (    0.36)    0.16%
string_rtns         97.107 (    0.21)     97.160 (    0.22)   -0.05%
disk_rd            626.890 (   18.25)    586.034 (    5.58)    6.97%

--
Gleb.

Gleb Natapov

author

2009-11-30 10:58 UTC #23 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 12 lines · Gleb Natapov

Quoted 8 lines · Christoph Lameter
Quoted 4 lines · Gleb Natapov
Quoted 2 lines · Christoph Lameter
This adds significant overhead for the !PREEMPT case adding lots of code in critical paths all over the place.
I want to measure it. Can you suggest benchmarks to try?
AIM9 (reaim9)?
Below are results for kernel 2.6.32-rc8 with and without the patch (only this single patch is applied).

Forgot to tell. The results are average between 5 different runs.

--
Gleb.

Peter Zijlstra

maintainer

2009-11-30 10:59 UTC #24 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 7 lines · Gleb Natapov

Quoted 6 lines · Gleb Natapov
Quoted 5 lines · Christoph Lameter
Quoted 3 lines · Gleb Natapov
Quoted 2 lines · Christoph Lameter
This adds significant overhead for the !PREEMPT case adding lots of code in critical paths all over the place.

Quoted 7 lines

Quoted 6 lines
Quoted 3 lines
Quoted 1 line
I want to measure it. Can you suggest benchmarks to try?
AIM9 (reaim9)?
Below are results for kernel 2.6.32-rc8 with and without the patch (only this single patch is applied).
Forgot to tell. The results are average between 5 different runs.

Would be good to also report the variance over those 5 runs, allows us to see if the difference is within the noise.

Avi Kivity

2009-11-30 11:01 UTC #25 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Quoted 5 lines · Peter Zijlstra

Quoted 2 lines
Forgot to tell. The results are average between 5 different runs.
Would be good to also report the variance over those 5 runs, allows us to see if the difference is within the noise.

That's the stddev column.

Signature · 7 lines

error compiling committee.c: too many arguments to function

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Peter Zijlstra

maintainer

2009-11-30 11:05 UTC #26 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

In reply to Peter Zijlstra

Quoted 10 lines · Peter Zijlstra

Quoted 9 lines · Gleb Natapov
Quoted 8 lines · Gleb Natapov
Quoted 7 lines · Christoph Lameter
Quoted 4 lines · Gleb Natapov
Quoted 2 lines · Christoph Lameter
This adds significant overhead for the !PREEMPT case adding lots of code in critical paths all over the place.
I want to measure it. Can you suggest benchmarks to try?

Quoted 8 lines

Quoted 5 lines
Quoted 4 lines
Quoted 1 line
AIM9 (reaim9)?
Below are results for kernel 2.6.32-rc8 with and without the patch (only this single patch is applied).
Forgot to tell. The results are average between 5 different runs.
Would be good to also report the variance over those 5 runs, allows us to see if the difference is within the noise.

Got pointed to the fact that there is a stddev column right there.

Must be Monday or something ;-)

Christoph Lameter

2009-11-30 16:23 UTC #27 lore raw

Re: [PATCH v2 10/12] Maintain preemptability count even for !CONFIG_PREEMPT kernels

Ok so there is some variance in tests as usual due to cacheline placement. But it seems that overall we are looking at a 1-2% regression.

Gleb Natapov

author

2009-11-23 14:06 UTC #28 lore raw

PATCH v2 11/12 Handle async PF in non preemptable context.

In reply to Gleb Natapov

If async page fault is received by idle task or when preemp_count is not zero guest cannot reschedule, so do sti; hlt and wait for page to be ready. vcpu can still process interrupts while it waits for the page to be ready.

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/kernel/kvm.c |   31 +++++++++++++++++++++++++++----
 1 files changed, 27 insertions(+), 4 deletions(-)

arch/x86/kernel/kvm.c +27 -4

index 09444c9..0836d9a 100644
@@ -63,6 +63,7 @@ struct kvm_task_sleep_node {
 	struct hlist_node link;
 	wait_queue_head_t wq;
 	u32 token;
+	int cpu;
 };
 
 static struct kvm_task_sleep_head {

@@ -91,6 +92,11 @@ static void apf_task_wait(struct task_struct *tsk, u32 token)
 	struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
 	struct kvm_task_sleep_node n, *e;
 	DEFINE_WAIT(wait);
+	int cpu, idle;
+
+	cpu = get_cpu();
+	idle = idle_cpu(cpu);
+	put_cpu();
 
 	spin_lock(&b->lock);
 	e = _find_apf_task(b, token);

@@ -105,15 +111,30 @@ static void apf_task_wait(struct task_struct *tsk, u32 token)
 	n.token = token;
 	init_waitqueue_head(&n.wq);
 	hlist_add_head(&n.link, &b->list);
+	if (idle || preempt_count() > 1)
+		n.cpu = smp_processor_id();
+	else
+		n.cpu = -1;
 	spin_unlock(&b->lock);
 
 	for (;;) {
-		prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
+		if (n.cpu < 0)
+			prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
 		if (hlist_unhashed(&n.link))
 			break;
-		schedule();
+
+		if (n.cpu < 0) {
+			schedule();
+		} else {
+			/*
+			 * We cannot reschedule. So halt.
+			 */
+			native_safe_halt();
+			local_irq_disable();
+		}
 	}
-	finish_wait(&n.wq, &wait);
+	if (n.cpu < 0)
+		finish_wait(&n.wq, &wait);
 
 	return;
 }

@@ -146,7 +167,9 @@ again:
 		hlist_add_head(&n->link, &b->list);
 	} else {
 		hlist_del_init(&n->link);
-		if (waitqueue_active(&n->wq))
+		if (n->cpu >= 0)
+			smp_send_reschedule(n->cpu);
+		else if (waitqueue_active(&n->wq))
 			wake_up(&n->wq);
 	}
 	spin_unlock(&b->lock);

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Gleb Natapov

author

2009-11-23 14:06 UTC #29 lore raw

PATCH v2 12/12 Send async PF when guest is not in userspace too.

In reply to Gleb Natapov

Signed-off-by Gleb Natapov <[email protected]>

---
 arch/x86/kvm/mmu.c |    5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

arch/x86/kvm/mmu.c +3 -2

index 1225c31..a538d82 100644
@@ -2204,10 +2204,11 @@ int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gva,
 
 static bool can_do_async_pf(struct kvm_vcpu *vcpu)
 {
-	if (!vcpu->arch.apf_data || kvm_event_needs_reinjection(vcpu))
+	if (!vcpu->arch.apf_data || kvm_event_needs_reinjection(vcpu) ||
+	    !kvm_x86_ops->interrupt_allowed(vcpu))
 		return false;
 
-	return !!kvm_x86_ops->get_cpl(vcpu);
+	return true;
 }
 
 static int tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t cr3, gva_t gpa,

Signature · 7 lines

1.6.5

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>