Keep cr3 cached in VMCS
v2 cover Keep cr3 cached in VMCS
This patchset keeps cr3 cached in the VMCS (GUEST_CR3) instead of syncing it every exit. Saves a VMREAD/VMWRITE on Intel with EPT enabled.
Avi Kivity (2):
KVM: Replace reads of vcpu->arch.cr3 by an accessor
KVM: Fetch guest cr3 from hardware on demand
v2: fix ept identity pagetable leak into guest cr3
arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/kvm_cache_regs.h | 7 +++++++ arch/x86/kvm/mmu.c | 6 +++--- arch/x86/kvm/svm.c | 13 +++++++++---- arch/x86/kvm/vmx.c | 30 +++++++++++++++++++++--------- arch/x86/kvm/x86.c | 19 ++++++++++--------- 6 files changed, 52 insertions(+), 25 deletions(-)
PATCH v2 1/2 KVM: Replace reads of vcpu->arch.cr3 by an accessor
This allows us to keep cr3 in the VMCS, later on.
--- arch/x86/kvm/kvm_cache_regs.h | 5 +++++ arch/x86/kvm/mmu.c | 6 +++--- arch/x86/kvm/svm.c | 8 ++++---- arch/x86/kvm/vmx.c | 7 ++++--- arch/x86/kvm/x86.c | 19 ++++++++++--------- 5 files changed, 26 insertions(+), 19 deletions(-)
arch/x86/kvm/kvm_cache_regs.h +5 -0
@@ -73,6 +73,11 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)return vcpu->arch.cr4 & mask; } +static inline ulong kvm_read_cr3 (struct kvm_vcpu *vcpu) +{ +return vcpu->arch.cr3; +} +static inline ulong kvm_read_cr4 (struct kvm_vcpu *vcpu) {return kvm_read_cr4_bits (vcpu, ~0UL );
arch/x86/kvm/mmu.c +3 -3
@@ -2726,13 +2726,13 @@ void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)static void paging_new_cr3 (struct kvm_vcpu *vcpu) { -pgprintk ("%s: cr3 %lx\n" , __func__, vcpu->arch.cr3); +pgprintk ("%s: cr3 %lx\n" , __func__,kvm_read_cr3 (vcpu));mmu_free_roots (vcpu); }static unsigned long get_cr3 (struct kvm_vcpu *vcpu) { -return vcpu->arch.cr3; +return kvm_read_cr3 (vcpu); } static void inject_page_fault(struct kvm_vcpu *vcpu,
@@ -3631,7 +3631,7 @@ static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,static int kvm_pv_mmu_flush_tlb (struct kvm_vcpu *vcpu) { - (void )kvm_set_cr3 (vcpu, vcpu->arch.cr3); + (void )kvm_set_cr3 (vcpu,kvm_read_cr3 (vcpu));return 1 ; }
arch/x86/kvm/svm.c +4 -4
@@ -1170,7 +1170,7 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)switch (reg) {case VCPU_EXREG_PDPTR :BUG_ON (!npt_enabled); -load_pdptrs (vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); +load_pdptrs (vcpu, vcpu->arch.walk_mmu,kvm_read_cr3 (vcpu));break ;default :BUG ();
@@ -2110,7 +2110,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->save.idtr = vmcb->save.idtr; nested_vmcb->save.efer = svm->vcpu.arch.efer; nested_vmcb->save.cr0 =kvm_read_cr0 (&svm->vcpu); - nested_vmcb->save.cr3 = svm->vcpu.arch.cr3; + nested_vmcb->save.cr3 =kvm_read_cr3 (&svm->vcpu); nested_vmcb->save.cr2 = vmcb->save.cr2; nested_vmcb->save.cr4 = svm->vcpu.arch.cr4; nested_vmcb->save.rflags = vmcb->save.rflags;
@@ -2305,7 +2305,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)if (npt_enabled) hsave->save.cr3 = vmcb->save.cr3;else - hsave->save.cr3 = svm->vcpu.arch.cr3; + hsave->save.cr3 =kvm_read_cr3 (&svm->vcpu);copy_vmcb_control_area (hsave, vmcb);
@@ -3587,7 +3587,7 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)mark_dirty (svm->vmcb,VMCB_NPT );/* Also sync guest cr3 here in case we live migrate */ - svm->vmcb->save.cr3 = vcpu->arch.cr3; + svm->vmcb->save.cr3 =kvm_read_cr3 (vcpu);mark_dirty (svm->vmcb,VMCB_CR );svm_flush_tlb (vcpu);
arch/x86/kvm/vmx.c +4 -3
@@ -1959,7 +1959,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)if (enable_ept) { eptp =construct_eptp (cr3);vmcs_write64 (EPT_POINTER , eptp); - guest_cr3 =is_paging (vcpu) ? vcpu->arch.cr3 : + guest_cr3 =is_paging (vcpu) ?kvm_read_cr3 (vcpu) : vcpu->kvm->arch.ept_identity_map_addr;ept_load_pdptrs (vcpu); }
@@ -3205,8 +3205,9 @@ static int handle_cr(struct kvm_vcpu *vcpu)case 1 :/*mov from cr*/ switch (cr) {case 3 : -kvm_register_write (vcpu, reg, vcpu->arch.cr3); -trace_kvm_cr_read (cr, vcpu->arch.cr3); + val =kvm_read_cr3 (vcpu); +kvm_register_write (vcpu, reg, val); +trace_kvm_cr_read (cr, val);skip_emulated_instruction (vcpu);return 1 ;case 8 :
arch/x86/kvm/x86.c +10 -9
@@ -464,8 +464,8 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu) (unsigned long *)&vcpu->arch.regs_avail))return true; - gfn = (vcpu->arch.cr3 & ~31u ) >>PAGE_SHIFT ; - offset = (vcpu->arch.cr3 & ~31u ) & (PAGE_SIZE -1 ); + gfn = (kvm_read_cr3 (vcpu) & ~31u ) >>PAGE_SHIFT ; + offset = (kvm_read_cr3 (vcpu) & ~31u ) & (PAGE_SIZE -1 ); r =kvm_read_nested_guest_page (vcpu, gfn, pdpte, offset,sizeof (pdpte),PFERR_USER_MASK |PFERR_WRITE_MASK );if (r <0 )
@@ -510,7 +510,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } else#endif if (is_pae (vcpu) && !load_pdptrs (vcpu, vcpu->arch.walk_mmu, - vcpu->arch.cr3)) +kvm_read_cr3 (vcpu)))return 1 ; }
@@ -602,7 +602,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)return 1 ; }else if (is_paging (vcpu) && (cr4 &X86_CR4_PAE ) && ((cr4 ^ old_cr4) & pdptr_bits) - && !load_pdptrs (vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3)) + && !load_pdptrs (vcpu, vcpu->arch.walk_mmu, +kvm_read_cr3 (vcpu)))return 1 ;if (cr4 &X86_CR4_VMXE )
@@ -622,7 +623,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr4);int kvm_set_cr3 (struct kvm_vcpu *vcpu,unsigned long cr3) { -if (cr3 == vcpu->arch.cr3 && !pdptrs_changed (vcpu)) { +if (cr3 ==kvm_read_cr3 (vcpu) && !pdptrs_changed (vcpu)) {kvm_mmu_sync_roots (vcpu);kvm_mmu_flush_tlb (vcpu);return 0 ;
@@ -4070,7 +4071,7 @@ static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu) value = vcpu->arch.cr2;break ;case 3 : - value = vcpu->arch.cr3; + value =kvm_read_cr3 (vcpu);break ;case 4 : value =kvm_read_cr4 (vcpu);
@@ -5504,7 +5505,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sregs->cr0 =kvm_read_cr0 (vcpu); sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = vcpu->arch.cr3; + sregs->cr3 =kvm_read_cr3 (vcpu); sregs->cr4 =kvm_read_cr4 (vcpu); sregs->cr8 =kvm_get_cr8 (vcpu); sregs->efer = vcpu->arch.efer;
@@ -5572,7 +5573,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, kvm_x86_ops->set_gdt (vcpu, &dt); vcpu->arch.cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; + mmu_reset_needed |=kvm_read_cr3 (vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3;kvm_set_cr8 (vcpu, sregs->cr8);
@@ -5590,7 +5591,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,if (sregs->cr4 &X86_CR4_OSXSAVE )update_cpuid (vcpu);if (!is_long_mode (vcpu) &&is_pae (vcpu)) { -load_pdptrs (vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); +load_pdptrs (vcpu, vcpu->arch.walk_mmu,kvm_read_cr3 (vcpu)); mmu_reset_needed =1 ; }
Signature · 1 line
1.7.1
PATCH v2 2/2 KVM: Fetch guest cr3 from hardware on demand
In reply to Avi KivityInstead of syncing the guest cr3 every exit, which is expensince on vmx with ept enabled, sync it only on demand.
--- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/kvm_cache_regs.h | 2 ++ arch/x86/kvm/svm.c | 5 +++++ arch/x86/kvm/vmx.c | 23 +++++++++++++++++------ 4 files changed, 26 insertions(+), 6 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -0
@@ -117,6 +117,7 @@ enum kvm_reg {enum kvm_reg_ex {VCPU_EXREG_PDPTR =NR_VCPU_REGS , +VCPU_EXREG_CR3 , };enum {
@@ -533,6 +534,7 @@ struct kvm_x86_ops { struct kvm_segment *var, int seg);void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu,int *db,int *l);void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); +void (*decache_cr3)(struct kvm_vcpu *vcpu);void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);void (*set_cr0)(struct kvm_vcpu *vcpu,unsigned long cr0);void (*set_cr3)(struct kvm_vcpu *vcpu,unsigned long cr3);
arch/x86/kvm/kvm_cache_regs.h +2 -0
@@ -75,6 +75,8 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)static inline ulong kvm_read_cr3 (struct kvm_vcpu *vcpu) { +if (!test_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail)) + kvm_x86_ops->decache_cr3 (vcpu);return vcpu->arch.cr3; }
arch/x86/kvm/svm.c +5 -0
@@ -1323,6 +1323,10 @@ static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) { } +static void svm_decache_cr3 (struct kvm_vcpu *vcpu) +{ +} +static void svm_decache_cr4_guest_bits (struct kvm_vcpu *vcpu) { }
@@ -3765,6 +3769,7 @@ static struct kvm_x86_ops svm_x86_ops = { .get_cpl = svm_get_cpl, .get_cs_db_l_bits = kvm_get_cs_db_l_bits, .decache_cr0_guest_bits = svm_decache_cr0_guest_bits, + .decache_cr3 = svm_decache_cr3, .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, .set_cr0 = svm_set_cr0, .set_cr3 = svm_set_cr3,
arch/x86/kvm/vmx.c +17 -6
@@ -180,6 +180,7 @@ static int init_rmode(struct kvm *kvm);static u64 construct_eptp (unsigned long root_hpa);static void kvm_cpu_vmxon (u64 addr);static void kvm_cpu_vmxoff (void ); +static void vmx_set_cr3 (struct kvm_vcpu *vcpu,unsigned long cr3); static DEFINE_PER_CPU(struct vmcs *, vmxarea); static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
@@ -1836,6 +1837,14 @@ static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) vcpu->arch.cr0 |=vmcs_readl (GUEST_CR0 ) & cr0_guest_owned_bits; } +static void vmx_decache_cr3 (struct kvm_vcpu *vcpu) +{ +/* Can only be called with EPT enabled */ +if (enable_ept &&is_paging (vcpu)) + vcpu->arch.cr3 =vmcs_readl (GUEST_CR3 ); +__set_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail); +} +static void vmx_decache_cr4_guest_bits (struct kvm_vcpu *vcpu) {ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
@@ -1879,6 +1888,8 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,unsigned long cr0, struct kvm_vcpu *vcpu) { + ulong cr3; +if (!(cr0 &X86_CR0_PG )) {/* From paging/starting to nonpaging */ vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
@@ -1893,8 +1904,11 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ) & ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING)); +/* Must fetch cr3 before updating cr0 */ + cr3 =kvm_read_cr3 (vcpu); vcpu->arch.cr0 = cr0;vmx_set_cr4 (vcpu,kvm_read_cr4 (vcpu)); +vmx_set_cr3 (vcpu, cr3); }if (!(cr0 &X86_CR0_WP ))
@@ -3734,11 +3748,6 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu)if (vmx->emulation_required && emulate_invalid_guest_state)return handle_invalid_guest_state (vcpu); -/* Access CR3 don't cause VMExit in paging mode, so we need -* to sync with guest real CR3. */ -if (enable_ept &&is_paging (vcpu)) - vcpu->arch.cr3 =vmcs_readl (GUEST_CR3 ); -if (exit_reason &VMX_EXIT_REASONS_FAILED_VMENTRY ) { vcpu->run->exit_reason =KVM_EXIT_FAIL_ENTRY ; vcpu->run->fail_entry.hardware_entry_failure_reason
@@ -4055,7 +4064,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) ); vcpu->arch.regs_avail = ~((1 <<VCPU_REGS_RIP ) | (1 <<VCPU_REGS_RSP ) - | (1 <<VCPU_EXREG_PDPTR )); + | (1 <<VCPU_EXREG_PDPTR ) + | (1 <<VCPU_EXREG_CR3 )); vcpu->arch.regs_dirty =0 ; vmx->idt_vectoring_info =vmcs_read32 (IDT_VECTORING_INFO_FIELD );
@@ -4322,6 +4332,7 @@ static struct kvm_x86_ops vmx_x86_ops = { .get_cpl = vmx_get_cpl, .get_cs_db_l_bits = vmx_get_cs_db_l_bits, .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits, + .decache_cr3 = vmx_decache_cr3, .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, .set_cr0 = vmx_set_cr0, .set_cr3 = vmx_set_cr3,
Signature · 1 line
1.7.1
Re: [PATCH v2 2/2] KVM: Fetch guest cr3 from hardware on demand
Quoted 36 lines · Avi Kivity
Instead of syncing the guest cr3 every exit, which is expensince on vmx with ept enabled, sync it only on demand.
Signed-off-by Avi Kivity <[email protected]>--- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/kvm_cache_regs.h | 2 ++ arch/x86/kvm/svm.c | 5 +++++ arch/x86/kvm/vmx.c | 23 +++++++++++++++++------ 4 files changed, 26 insertions(+), 6 deletions(-)arch/x86/include/asm/kvm_host.h +2
-0@@ -117,6 +117,7 @@ enum kvm_reg {enum kvm_reg_ex {VCPU_EXREG_PDPTR =NR_VCPU_REGS , +VCPU_EXREG_CR3 , };enum {@@ -533,6 +534,7 @@ struct kvm_x86_ops { struct kvm_segment *var, int seg);void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu,int *db,int *l);void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); +void (*decache_cr3)(struct kvm_vcpu *vcpu);void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);void (*set_cr0)(struct kvm_vcpu *vcpu,unsigned long cr0);void (*set_cr3)(struct kvm_vcpu *vcpu,unsigned long cr3);arch/x86/kvm/kvm_cache_regs.h +0
-0
arch/x86/kvm/kvm_cache_regs.h static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask) · line 81
Quoted 7 lines
arch/x86/kvm/kvm_cache_regs.h +2
-0@@ -75,6 +75,8 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)static inline ulong kvm_read_cr3 (struct kvm_vcpu *vcpu) { +if (!test_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail)) + kvm_x86_ops->decache_cr3 (vcpu);return vcpu->arch.cr3; }
Should mark register available on vcpu->arch.cr3 assignment (kvm_set_cr3, kvm_set_sregs, etc), so that subsequent cr3 read does not use decache value.
Re: [PATCH v2 2/2] KVM: Fetch guest cr3 from hardware on demand
Quoted 12 lines · Marcelo Tosatti
Quoted 10 lines · Avi Kivity
Instead of syncing the guest cr3 every exit, which is expensince on vmx with ept enabled, sync it only on demand.
static inline ulong
kvm_read_cr3(struct kvm_vcpu *vcpu){
+ if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
+kvm_x86_ops->decache_cr3(vcpu);
returnvcpu->arch.cr3;
}
Quoted 3 lines
Should mark register available on
vcpu->arch.cr3assignment (kvm_set_cr3,kvm_set_sregs, etc), so that subsequent cr3 read does not use decache value.
True. Will fix and repost.
(it's not a correctness problem, since the cached value will match GUEST_CR3, but it's nicer)
Signature · 1 line
error compiling committee.c: too many arguments to function