KVM: Get rid of KVM_REQ_KICK
PATCH 1/1 KVM: Get rid of KVM_REQ_KICK
KVM_REQ_KICK poisons vcpu->requests by having a bit set during normal operation. This causes the fast path check for a clear vcpu->requests to fail all the time, triggering tons of atomic operations.
Fix by replacing KVM_REQ_KICK with a vcpu->guest_mode atomic.
--- arch/x86/kvm/x86.c | 17 ++++++++++------- include/linux/kvm_host.h | 1 + 2 files changed, 11 insertions(+), 7 deletions(-)
arch/x86/kvm/x86.c +10 -7
@@ -4499,13 +4499,15 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)if (vcpu->fpu_active)kvm_load_guest_fpu (vcpu); -local_irq_disable (); +atomic_set (&vcpu->guest_mode,1 ); +smp_wmb (); -clear_bit (KVM_REQ_KICK , &vcpu->requests); -smp_mb__after_clear_bit (); +local_irq_disable (); -if (vcpu->requests ||need_resched () ||signal_pending (current)) { -set_bit (KVM_REQ_KICK , &vcpu->requests); +if (!atomic_read (&vcpu->guest_mode) || vcpu->requests + ||need_resched () ||signal_pending (current)) { +atomic_set (&vcpu->guest_mode,0 ); +smp_wmb ();local_irq_enable ();preempt_enable (); r =1 ;
@@ -4550,7 +4552,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)if (hw_breakpoint_active ())hw_breakpoint_restore (); -set_bit (KVM_REQ_KICK , &vcpu->requests); +atomic_set (&vcpu->guest_mode,0 ); +smp_wmb ();local_irq_enable (); ++vcpu->stat.exits;
@@ -5470,7 +5473,7 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu) me =get_cpu ();if (cpu != me && (unsigned )cpu < nr_cpu_ids &&cpu_online (cpu)) -if (!test_and_set_bit (KVM_REQ_KICK , &vcpu->requests)) +if (atomic_xchg (&vcpu->guest_mode,0 ))smp_send_reschedule (cpu);put_cpu (); }
include/linux/kvm_host.h +1 -0
@@ -81,6 +81,7 @@ struct kvm_vcpu {int vcpu_id;struct mutex mutex;int cpu; +atomic_t guest_mode;struct kvm_run *run;unsigned long requests;unsigned long guest_debug;
Signature · 1 line
1.7.0.4
Quoted 3 lines · Avi Kivity
KVM_REQ_KICKpoisonsvcpu->requestsby having a bit set during normal operation. This causes the fast path check for a clearvcpu->requeststo fail all the time, triggering tons of atomic operations.
Avi,
Do you have numbers?
Quoted 13 lines
Fix by replacing
KVM_REQ_KICKwith avcpu->guest_modeatomic.Signed-off-by Avi Kivity <[email protected]>--- arch/x86/kvm/x86.c | 17 ++++++++++------- include/linux/kvm_host.h | 1 + 2 files changed, 11 insertions(+), 7 deletions(-)arch/x86/kvm/x86.c +0
-0
arch/x86/kvm/x86.c static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · line 4503
Quoted 6 lines
arch/x86/kvm/x86.c +2
-1@@ -4499,13 +4499,15 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)if (vcpu->fpu_active)kvm_load_guest_fpu (vcpu); -local_irq_disable (); +atomic_set (&vcpu->guest_mode,1 ); +smp_wmb ();
IPI can trigger here?
Quoted 51 lines
arch/x86/kvm/x86.c +8
-6-clear_bit (KVM_REQ_KICK , &vcpu->requests); -smp_mb__after_clear_bit (); +local_irq_disable (); -if (vcpu->requests ||need_resched () ||signal_pending (current)) { -set_bit (KVM_REQ_KICK , &vcpu->requests); +if (!atomic_read (&vcpu->guest_mode) || vcpu->requests + ||need_resched () ||signal_pending (current)) { +atomic_set (&vcpu->guest_mode,0 ); +smp_wmb ();local_irq_enable ();preempt_enable (); r =1 ;@@ -4550,7 +4552,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)if (hw_breakpoint_active ())hw_breakpoint_restore (); -set_bit (KVM_REQ_KICK , &vcpu->requests); +atomic_set (&vcpu->guest_mode,0 ); +smp_wmb ();local_irq_enable (); ++vcpu->stat.exits;@@ -5470,7 +5473,7 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu) me =get_cpu ();if (cpu != me && (unsigned )cpu < nr_cpu_ids &&cpu_online (cpu)) -if (!test_and_set_bit (KVM_REQ_KICK , &vcpu->requests)) +if (atomic_xchg (&vcpu->guest_mode,0 ))smp_send_reschedule (cpu);put_cpu (); }include/linux/kvm_host.h +1
-0@@ -81,6 +81,7 @@ struct kvm_vcpu {int vcpu_id;struct mutex mutex;int cpu; +atomic_t guest_mode;struct kvm_run *run;unsigned long requests;unsigned long guest_debug;Signature · 6 lines
1.7.0.4 -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 10 lines · Marcelo Tosatti
Quoted 4 lines · Avi Kivity
KVM_REQ_KICKpoisonsvcpu->requestsby having a bit set during normal operation. This causes the fast path check for a clearvcpu->requeststo fail all the time, triggering tons of atomic operations.Avi,
Do you have numbers?
Forgot to post, was about 100 cycles (I expected more, all those atomics really show up in the profile).
Quoted 16 lines
Quoted 16 lines
Fix by replacing
KVM_REQ_KICKwith avcpu->guest_modeatomic.Signed-off-by Avi Kivity<[email protected]>--- arch/x86/kvm/x86.c | 17 ++++++++++------- include/linux/kvm_host.h | 1 + 2 files changed, 11 insertions(+), 7 deletions(-)arch/x86/kvm/x86.c +0
-0@@ -4499,13 +4499,15 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)if (vcpu->fpu_active)kvm_load_guest_fpu (vcpu);
Quoted 6 lines
arch/x86/kvm/x86.c static int vcpu_enter_guest(struct kvm_vcpu *vcpu) · line 4504
Quoted 4 lines
arch/x86/kvm/x86.c +2
-1-local_irq_disable (); +atomic_set (&vcpu->guest_mode,1 ); +smp_wmb ();IPI can trigger here?
It can...
Quoted 10 lines
arch/x86/kvm/x86.c static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
Quoted 10 lines
arch/x86/kvm/x86.c +3
-4-clear_bit (KVM_REQ_KICK ,&vcpu->requests); -smp_mb__after_clear_bit (); +local_irq_disable (); -if (vcpu->requests ||need_resched () ||signal_pending (current)) { -set_bit (KVM_REQ_KICK ,&vcpu->requests); +if (!atomic_read (&vcpu->guest_mode) || vcpu->requests + ||need_resched () ||signal_pending (current)) {
... and we'll detect that guest_mode was cleared and go back.
Quoted 17 lines
Quoted 17 lines
arch/x86/kvm/x86.c +2
-0+atomic_set (&vcpu->guest_mode,0 ); +smp_wmb ();local_irq_enable ();preempt_enable ();r = 1;arch/x86/kvm/x86.c +2
-1@@ -4550,7 +4552,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)if (hw_breakpoint_active ())hw_breakpoint_restore (); -set_bit (KVM_REQ_KICK ,&vcpu->requests); +atomic_set (&vcpu->guest_mode,0 ); +smp_wmb ();local_irq_enable (); ++vcpu->stat.exits;@@ -5470,7 +5473,7 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
Quoted 7 lines
arch/x86/kvm/x86.c void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
Quoted 7 lines
arch/x86/kvm/x86.c +1
-1me =get_cpu ();if (cpu != me&& (unsigned )cpu< nr_cpu_ids&&cpu_online (cpu)) -if (!test_and_set_bit (KVM_REQ_KICK ,&vcpu->requests)) +if (atomic_xchg (&vcpu->guest_mode,0 ))smp_send_reschedule (cpu);put_cpu ();
The atomic_xchg() does the trick.
Signature · 1 line
error compiling committee.c: too many arguments to function
Quoted 12 lines · Avi Kivity
Quoted 8 lines
arch/x86/kvm/x86.c void kvm_vcpu_kick(struct kvm_vcpu *vcpu) · line 5478
Quoted 8 lines
arch/x86/kvm/x86.c +1
-1@@ -5470,7 +5473,7 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu) me =get_cpu ();if (cpu != me&& (unsigned )cpu< nr_cpu_ids&&cpu_online (cpu)) -if (!test_and_set_bit (KVM_REQ_KICK ,&vcpu->requests)) +if (atomic_xchg (&vcpu->guest_mode,0 ))smp_send_reschedule (cpu);put_cpu ();The
atomic_xchg()does the trick.
btw, things like this should go into locking.txt. It's definitely tricky.
Signature · 1 line
error compiling committee.c: too many arguments to function
Quoted 11 lines · Avi Kivity
KVM_REQ_KICKpoisonsvcpu->requestsby having a bit set during normal operation. This causes the fast path check for a clearvcpu->requeststo fail all the time, triggering tons of atomic operations.Fix by replacing
KVM_REQ_KICKwith avcpu->guest_modeatomic.Signed-off-by Avi Kivity <[email protected]>--- arch/x86/kvm/x86.c | 17 ++++++++++------- include/linux/kvm_host.h | 1 + 2 files changed, 11 insertions(+), 7 deletions(-)