Keep cr3 cached in VMCS
v3 cover Keep cr3 cached in VMCS
This patchset keeps cr3 cached in the VMCS (GUEST_CR3) instead of syncing it every exit. Saves a VMREAD/VMWRITE on Intel with EPT enabled.
Avi Kivity (2):
KVM: Replace reads of vcpu->arch.cr3 by an accessor
KVM: Fetch guest cr3 from hardware on demand
v3: mark cr3 as available when we write it
remove misleading comment in vmx_decache_cr3()
v2: fix ept identity pagetable leak into guest cr3
arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/kvm_cache_regs.h | 7 +++++++ arch/x86/kvm/mmu.c | 6 +++--- arch/x86/kvm/svm.c | 13 +++++++++---- arch/x86/kvm/vmx.c | 29 ++++++++++++++++++++--------- arch/x86/kvm/x86.c | 21 ++++++++++++--------- 6 files changed, 53 insertions(+), 25 deletions(-)
PATCH v3 1/2 KVM: Replace reads of vcpu->arch.cr3 by an accessor
This allows us to keep cr3 in the VMCS, later on.
Signed-off-by Avi Kivity <[email protected]>
--- arch/x86/kvm/kvm_cache_regs.h | 5 +++++ arch/x86/kvm/mmu.c | 6 +++--- arch/x86/kvm/svm.c | 8 ++++---- arch/x86/kvm/vmx.c | 7 ++++--- arch/x86/kvm/x86.c | 19 ++++++++++--------- 5 files changed, 26 insertions(+), 19 deletions(-)
arch/x86/kvm/kvm_cache_regs.h +5 -0
@@ -73,6 +73,11 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)return vcpu->arch.cr4 & mask; } +static inline ulong kvm_read_cr3 (struct kvm_vcpu *vcpu) +{ +return vcpu->arch.cr3; +} +static inline ulong kvm_read_cr4 (struct kvm_vcpu *vcpu) {return kvm_read_cr4_bits (vcpu, ~0UL );
arch/x86/kvm/mmu.c +3 -3
@@ -2726,13 +2726,13 @@ void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)static void paging_new_cr3 (struct kvm_vcpu *vcpu) { -pgprintk ("%s: cr3 %lx\n" , __func__, vcpu->arch.cr3); +pgprintk ("%s: cr3 %lx\n" , __func__,kvm_read_cr3 (vcpu));mmu_free_roots (vcpu); }static unsigned long get_cr3 (struct kvm_vcpu *vcpu) { -return vcpu->arch.cr3; +return kvm_read_cr3 (vcpu); } static void inject_page_fault(struct kvm_vcpu *vcpu,
@@ -3631,7 +3631,7 @@ static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,static int kvm_pv_mmu_flush_tlb (struct kvm_vcpu *vcpu) { - (void )kvm_set_cr3 (vcpu, vcpu->arch.cr3); + (void )kvm_set_cr3 (vcpu,kvm_read_cr3 (vcpu));return 1 ; }
arch/x86/kvm/svm.c +4 -4
@@ -1170,7 +1170,7 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)switch (reg) {case VCPU_EXREG_PDPTR :BUG_ON (!npt_enabled); -load_pdptrs (vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); +load_pdptrs (vcpu, vcpu->arch.walk_mmu,kvm_read_cr3 (vcpu));break ;default :BUG ();
@@ -2110,7 +2110,7 @@ static int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->save.idtr = vmcb->save.idtr; nested_vmcb->save.efer = svm->vcpu.arch.efer; nested_vmcb->save.cr0 =kvm_read_cr0 (&svm->vcpu); - nested_vmcb->save.cr3 = svm->vcpu.arch.cr3; + nested_vmcb->save.cr3 =kvm_read_cr3 (&svm->vcpu); nested_vmcb->save.cr2 = vmcb->save.cr2; nested_vmcb->save.cr4 = svm->vcpu.arch.cr4; nested_vmcb->save.rflags = vmcb->save.rflags;
@@ -2305,7 +2305,7 @@ static bool nested_svm_vmrun(struct vcpu_svm *svm)if (npt_enabled) hsave->save.cr3 = vmcb->save.cr3;else - hsave->save.cr3 = svm->vcpu.arch.cr3; + hsave->save.cr3 =kvm_read_cr3 (&svm->vcpu);copy_vmcb_control_area (hsave, vmcb);
@@ -3587,7 +3587,7 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)mark_dirty (svm->vmcb,VMCB_NPT );/* Also sync guest cr3 here in case we live migrate */ - svm->vmcb->save.cr3 = vcpu->arch.cr3; + svm->vmcb->save.cr3 =kvm_read_cr3 (vcpu);mark_dirty (svm->vmcb,VMCB_CR );svm_flush_tlb (vcpu);
arch/x86/kvm/vmx.c +4 -3
@@ -1959,7 +1959,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)if (enable_ept) { eptp =construct_eptp (cr3);vmcs_write64 (EPT_POINTER , eptp); - guest_cr3 =is_paging (vcpu) ? vcpu->arch.cr3 : + guest_cr3 =is_paging (vcpu) ?kvm_read_cr3 (vcpu) : vcpu->kvm->arch.ept_identity_map_addr;ept_load_pdptrs (vcpu); }
@@ -3205,8 +3205,9 @@ static int handle_cr(struct kvm_vcpu *vcpu)case 1 :/*mov from cr*/ switch (cr) {case 3 : -kvm_register_write (vcpu, reg, vcpu->arch.cr3); -trace_kvm_cr_read (cr, vcpu->arch.cr3); + val =kvm_read_cr3 (vcpu); +kvm_register_write (vcpu, reg, val); +trace_kvm_cr_read (cr, val);skip_emulated_instruction (vcpu);return 1 ;case 8 :
arch/x86/kvm/x86.c +10 -9
@@ -464,8 +464,8 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu) (unsigned long *)&vcpu->arch.regs_avail))return true; - gfn = (vcpu->arch.cr3 & ~31u ) >>PAGE_SHIFT ; - offset = (vcpu->arch.cr3 & ~31u ) & (PAGE_SIZE -1 ); + gfn = (kvm_read_cr3 (vcpu) & ~31u ) >>PAGE_SHIFT ; + offset = (kvm_read_cr3 (vcpu) & ~31u ) & (PAGE_SIZE -1 ); r =kvm_read_nested_guest_page (vcpu, gfn, pdpte, offset,sizeof (pdpte),PFERR_USER_MASK |PFERR_WRITE_MASK );if (r <0 )
@@ -510,7 +510,7 @@ int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } else#endif if (is_pae (vcpu) && !load_pdptrs (vcpu, vcpu->arch.walk_mmu, - vcpu->arch.cr3)) +kvm_read_cr3 (vcpu)))return 1 ; }
@@ -602,7 +602,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)return 1 ; }else if (is_paging (vcpu) && (cr4 &X86_CR4_PAE ) && ((cr4 ^ old_cr4) & pdptr_bits) - && !load_pdptrs (vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3)) + && !load_pdptrs (vcpu, vcpu->arch.walk_mmu, +kvm_read_cr3 (vcpu)))return 1 ;if (cr4 &X86_CR4_VMXE )
@@ -622,7 +623,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cr4);int kvm_set_cr3 (struct kvm_vcpu *vcpu,unsigned long cr3) { -if (cr3 == vcpu->arch.cr3 && !pdptrs_changed (vcpu)) { +if (cr3 ==kvm_read_cr3 (vcpu) && !pdptrs_changed (vcpu)) {kvm_mmu_sync_roots (vcpu);kvm_mmu_flush_tlb (vcpu);return 0 ;
@@ -4070,7 +4071,7 @@ static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu) value = vcpu->arch.cr2;break ;case 3 : - value = vcpu->arch.cr3; + value =kvm_read_cr3 (vcpu);break ;case 4 : value =kvm_read_cr4 (vcpu);
@@ -5504,7 +5505,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sregs->cr0 =kvm_read_cr0 (vcpu); sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = vcpu->arch.cr3; + sregs->cr3 =kvm_read_cr3 (vcpu); sregs->cr4 =kvm_read_cr4 (vcpu); sregs->cr8 =kvm_get_cr8 (vcpu); sregs->efer = vcpu->arch.efer;
@@ -5572,7 +5573,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, kvm_x86_ops->set_gdt (vcpu, &dt); vcpu->arch.cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; + mmu_reset_needed |=kvm_read_cr3 (vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3;kvm_set_cr8 (vcpu, sregs->cr8);
@@ -5590,7 +5591,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,if (sregs->cr4 &X86_CR4_OSXSAVE )update_cpuid (vcpu);if (!is_long_mode (vcpu) &&is_pae (vcpu)) { -load_pdptrs (vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3); +load_pdptrs (vcpu, vcpu->arch.walk_mmu,kvm_read_cr3 (vcpu)); mmu_reset_needed =1 ; }
Signature · 1 line
1.7.1
PATCH v3 2/2 KVM: Fetch guest cr3 from hardware on demand
In reply to Avi KivityInstead of syncing the guest cr3 every exit, which is expensince on vmx with ept enabled, sync it only on demand.
Signed-off-by Avi Kivity <[email protected]>
--- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/kvm_cache_regs.h | 2 ++ arch/x86/kvm/svm.c | 5 +++++ arch/x86/kvm/vmx.c | 22 ++++++++++++++++------ arch/x86/kvm/x86.c | 2 ++ 5 files changed, 27 insertions(+), 6 deletions(-)
arch/x86/include/asm/kvm_host.h +2 -0
@@ -117,6 +117,7 @@ enum kvm_reg {enum kvm_reg_ex {VCPU_EXREG_PDPTR =NR_VCPU_REGS , +VCPU_EXREG_CR3 , };enum {
@@ -533,6 +534,7 @@ struct kvm_x86_ops { struct kvm_segment *var, int seg);void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu,int *db,int *l);void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); +void (*decache_cr3)(struct kvm_vcpu *vcpu);void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);void (*set_cr0)(struct kvm_vcpu *vcpu,unsigned long cr0);void (*set_cr3)(struct kvm_vcpu *vcpu,unsigned long cr3);
arch/x86/kvm/kvm_cache_regs.h +2 -0
@@ -75,6 +75,8 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)static inline ulong kvm_read_cr3 (struct kvm_vcpu *vcpu) { +if (!test_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail)) + kvm_x86_ops->decache_cr3 (vcpu);return vcpu->arch.cr3; }
arch/x86/kvm/svm.c +5 -0
@@ -1323,6 +1323,10 @@ static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) { } +static void svm_decache_cr3 (struct kvm_vcpu *vcpu) +{ +} +static void svm_decache_cr4_guest_bits (struct kvm_vcpu *vcpu) { }
@@ -3765,6 +3769,7 @@ static struct kvm_x86_ops svm_x86_ops = { .get_cpl = svm_get_cpl, .get_cs_db_l_bits = kvm_get_cs_db_l_bits, .decache_cr0_guest_bits = svm_decache_cr0_guest_bits, + .decache_cr3 = svm_decache_cr3, .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, .set_cr0 = svm_set_cr0, .set_cr3 = svm_set_cr3,
arch/x86/kvm/vmx.c +16 -6
@@ -180,6 +180,7 @@ static int init_rmode(struct kvm *kvm);static u64 construct_eptp (unsigned long root_hpa);static void kvm_cpu_vmxon (u64 addr);static void kvm_cpu_vmxoff (void ); +static void vmx_set_cr3 (struct kvm_vcpu *vcpu,unsigned long cr3); static DEFINE_PER_CPU(struct vmcs *, vmxarea); static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
@@ -1836,6 +1837,13 @@ static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) vcpu->arch.cr0 |=vmcs_readl (GUEST_CR0 ) & cr0_guest_owned_bits; } +static void vmx_decache_cr3 (struct kvm_vcpu *vcpu) +{ +if (enable_ept &&is_paging (vcpu)) + vcpu->arch.cr3 =vmcs_readl (GUEST_CR3 ); +__set_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail); +} +static void vmx_decache_cr4_guest_bits (struct kvm_vcpu *vcpu) {ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
@@ -1879,6 +1887,8 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,unsigned long cr0, struct kvm_vcpu *vcpu) { + ulong cr3; +if (!(cr0 &X86_CR0_PG )) {/* From paging/starting to nonpaging */ vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
@@ -1893,8 +1903,11 @@ static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,vmcs_read32 (CPU_BASED_VM_EXEC_CONTROL ) & ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING)); +/* Must fetch cr3 before updating cr0 */ + cr3 =kvm_read_cr3 (vcpu); vcpu->arch.cr0 = cr0;vmx_set_cr4 (vcpu,kvm_read_cr4 (vcpu)); +vmx_set_cr3 (vcpu, cr3); }if (!(cr0 &X86_CR0_WP ))
@@ -3734,11 +3747,6 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu)if (vmx->emulation_required && emulate_invalid_guest_state)return handle_invalid_guest_state (vcpu); -/* Access CR3 don't cause VMExit in paging mode, so we need -* to sync with guest real CR3. */ -if (enable_ept &&is_paging (vcpu)) - vcpu->arch.cr3 =vmcs_readl (GUEST_CR3 ); -if (exit_reason &VMX_EXIT_REASONS_FAILED_VMENTRY ) { vcpu->run->exit_reason =KVM_EXIT_FAIL_ENTRY ; vcpu->run->fail_entry.hardware_entry_failure_reason
@@ -4055,7 +4063,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) ); vcpu->arch.regs_avail = ~((1 <<VCPU_REGS_RIP ) | (1 <<VCPU_REGS_RSP ) - | (1 <<VCPU_EXREG_PDPTR )); + | (1 <<VCPU_EXREG_PDPTR ) + | (1 <<VCPU_EXREG_CR3 )); vcpu->arch.regs_dirty =0 ; vmx->idt_vectoring_info =vmcs_read32 (IDT_VECTORING_INFO_FIELD );
@@ -4322,6 +4331,7 @@ static struct kvm_x86_ops vmx_x86_ops = { .get_cpl = vmx_get_cpl, .get_cs_db_l_bits = vmx_get_cs_db_l_bits, .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits, + .decache_cr3 = vmx_decache_cr3, .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, .set_cr0 = vmx_set_cr0, .set_cr3 = vmx_set_cr3,
arch/x86/kvm/x86.c +2 -0
@@ -658,6 +658,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)if (unlikely (!gfn_to_memslot (vcpu->kvm, cr3 >>PAGE_SHIFT )))return 1 ; vcpu->arch.cr3 = cr3; +__set_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail); vcpu->arch.mmu.new_cr3 (vcpu);return 0 ; }
@@ -5575,6 +5576,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, vcpu->arch.cr2 = sregs->cr2; mmu_reset_needed |=kvm_read_cr3 (vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3; +__set_bit (VCPU_EXREG_CR3 , (ulong *)&vcpu->arch.regs_avail);kvm_set_cr8 (vcpu, sregs->cr8);
Signature · 1 line
1.7.1