KVM: VMX/SVM: use the same SPEC_CTRL assembly code
Paolo Bonzini
maintainerv3 cover KVM: VMX/SVM: use the same SPEC_CTRL assembly code
As discussed in the APX thread, this series moves the SPEC_CTRL handling for VMX entirely to vmenter.S, and while at it improves the code reuse between SVM and VMX.
The prototype of __vmx_vcpu_run() and __svm_vcpu_run() becomes the same, with a set of bit flags for the second argument (which can be augmented with an extra bit for whether to save/restore APX). The register allocation also becomes very similar, with %edi/%rdi pointing to the vmx (resp. svm) argument (patches 1-4).
Thanks to this, the code to restore the guest SPEC_CTRL value is the same up to the register names and can be dropped into vmx/vmenter.S. On the vmexit side the only change required is support for X86_FEATURE_KERNEL_IBRS, which is a single ALTERNATIVE macro.
This also introduces vmenter.h, which hosts common code between vmx/vmenter.S and svm/vmenter.S, and which will be augmented further by the APX enablement series.
Paolo
v2->v3:
- add extra patch so that it actually works [Chang S. Bae]
- fix incorrect reference to KVM_ENTER_RUN_* in comments
- preserve LFENCE in the eIBRS case [sashiko]
v1->v2:
- fix "Reload @vmx to RAX" comment [Binbin]
- fix polarity of KVM_ENTER_SAVE_SPEC_CTRL in SVM, which would have caused problems when the macros are reused for VMX
- adjust prototypes of __svm_vcpu_run and __svm_sev_es_vcpu_run
- fix order of ALTERNATIVE arguments for X86_FEATURE_KERNEL_IBRS. Add comment
- Leave @flags and @vmx on the stack during RESTORE_HOST_SPEC_CTRL_BODY. [Uros, Binbin]
Paolo Bonzini (8):
KVM: VMX: remove regs argument of __vmx_vcpu_run
KVM: VMX: more cleanups to __vmx_vcpu_run
KVM: SVM: prepare for making SPEC_CTRL switch common with VMX
KVM: SVM: pass struct vcpu_svm to msr_write_intercepted
KVM: SVM: adopt the same VMX_RUN_* flags as VMX
KVM: SVM: extract RESTORE_*_SPEC_CTRL_BODY out of svm/vmenter.S
KVM: VMX: switch to RESTORE_GUEST_SPEC_CTRL_BODY
KVM: VMX: replace vmx_spec_ctrl_restore_host with
RESTORE_HOST_SPEC_CTRL_BODY
arch/x86/kvm/kvm-asm-offsets.c | 1 + arch/x86/kvm/svm/svm.c | 23 +++-- arch/x86/kvm/svm/svm.h | 4 +- arch/x86/kvm/svm/vmenter.S | 103 +++++------------- arch/x86/kvm/vmenter.h | 80 ++++++++++++++ arch/x86/kvm/vmx/run_flags.h | 9 -- arch/x86/kvm/vmx/vmenter.S | 184 +++++++++++++++------------------ arch/x86/kvm/vmx/vmx.c | 39 ++----- arch/x86/kvm/vmx/vmx.h | 7 +- 9 files changed, 212 insertions(+), 238 deletions(-) create mode 100644 arch/x86/kvm/vmenter.h delete mode 100644 arch/x86/kvm/vmx/run_flags.h
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 1/8 KVM: VMX: remove regs argument of __vmx_vcpu_run
Registers are reachable through vcpu_vmx, no need to pass a separate pointer to the regs[] array.
No functional change intended.
--- arch/x86/kvm/kvm-asm-offsets.c | 1 + arch/x86/kvm/vmx/vmenter.S | 58 +++++++++++++++------------------- arch/x86/kvm/vmx/vmx.c | 3 +- arch/x86/kvm/vmx/vmx.h | 3 +- 4 files changed, 28 insertions(+), 37 deletions(-)
arch/x86/kvm/kvm-asm-offsets.c +1 -0
@@ -24,6 +24,7 @@ static void __used common(void)if (IS_ENABLED (CONFIG_KVM_INTEL )) {BLANK (); +OFFSET (VMX_vcpu_arch_regs, vcpu_vmx, vcpu.arch.regs);OFFSET (VMX_spec_ctrl, vcpu_vmx, spec_ctrl); } }
arch/x86/kvm/vmx/vmenter.S +25 -33
127 lines not on this page · raw message
arch/x86/kvm/vmx/vmx.c +1 -2
@@ -7468,8 +7468,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,if (vcpu->arch.cr2 !=native_read_cr2 ())native_write_cr2 (vcpu->arch.cr2); - vmx->fail =__vmx_vcpu_run (vmx, (unsigned long *)&vcpu->arch.regs, - flags); + vmx->fail =__vmx_vcpu_run (vmx, flags); vcpu->arch.cr2 =native_read_cr2 (); vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET ;
arch/x86/kvm/vmx/vmx.h +1 -2
@@ -370,8 +370,7 @@ void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu);void vmx_update_host_rsp (struct vcpu_vmx *vmx,unsigned long host_rsp);void vmx_spec_ctrl_restore_host (struct vcpu_vmx *vmx,unsigned int flags);unsigned int __vmx_vcpu_run_flags (struct vcpu_vmx *vmx); -bool __vmx_vcpu_run (struct vcpu_vmx *vmx,unsigned long *regs, -unsigned int flags); +bool __vmx_vcpu_run (struct vcpu_vmx *vmx,unsigned int flags);void vmx_ept_load_pdptrs (struct kvm_vcpu *vcpu);void vmx_set_intercept_for_msr (struct kvm_vcpu *vcpu,u32 msr,int type,bool set);
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 2/8 KVM: VMX: more cleanups to __vmx_vcpu_run
In reply to Paolo BonziniSlightly improve register allocation, loading vmx only once before vmlaunch/vmresume.
This also makes the code slightly more similar to the one for AMD processors, in that both keep the pointer to struct vcpu_vmx or vcpu_svm in %rdi. The code for restoring the guest value of SPEC_CTRL is also the same for Intel and AMD.
--- arch/x86/kvm/vmx/vmenter.S | 85 ++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 44 deletions(-)
arch/x86/kvm/vmx/vmenter.S +41 -44
@@ -98,11 +98,11 @@ SYM_FUNC_START(__vmx_vcpu_run) lea (%_ASM_SP), %_ASM_ARG2 call vmx_update_host_rsp - ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL - /* Reload @vmx, _ASM_ARG1 may be modified by vmx_update_host_rsp(). */ mov WORD_SIZE(%_ASM_SP), %_ASM_DI + ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL + /* * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the * host's, write the MSR.
@@ -122,9 +122,9 @@ SYM_FUNC_START(__vmx_vcpu_run) mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx xor %eax, %ecx mov VMX_spec_ctrl + 4(%edi), %edx - mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edi - xor %edx, %edi - or %edi, %ecx + mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi + xor %edx, %esi + or %esi, %ecx je .Lspec_ctrl_done #endif mov $MSR_IA32_SPEC_CTRL, %ecx
@@ -137,28 +137,25 @@ SYM_FUNC_START(__vmx_vcpu_run) * an LFENCE to stop speculation from skipping the wrmsr. */ - /* Load @vmx to RAX. */ - mov WORD_SIZE(%_ASM_SP), %_ASM_AX - /* Load guest registers. Don't clobber flags. */ - mov VCPU_RCX(%_ASM_AX), %_ASM_CX - mov VCPU_RDX(%_ASM_AX), %_ASM_DX - mov VCPU_RBX(%_ASM_AX), %_ASM_BX - mov VCPU_RBP(%_ASM_AX), %_ASM_BP - mov VCPU_RSI(%_ASM_AX), %_ASM_SI - mov VCPU_RDI(%_ASM_AX), %_ASM_DI + mov VCPU_RAX(%_ASM_DI), %_ASM_AX + mov VCPU_RCX(%_ASM_DI), %_ASM_CX + mov VCPU_RDX(%_ASM_DI), %_ASM_DX + mov VCPU_RBX(%_ASM_DI), %_ASM_BX + mov VCPU_RBP(%_ASM_DI), %_ASM_BP + mov VCPU_RSI(%_ASM_DI), %_ASM_SI #ifdef CONFIG_X86_64 - mov VCPU_R8 (%_ASM_AX), %r8 - mov VCPU_R9 (%_ASM_AX), %r9 - mov VCPU_R10(%_ASM_AX), %r10 - mov VCPU_R11(%_ASM_AX), %r11 - mov VCPU_R12(%_ASM_AX), %r12 - mov VCPU_R13(%_ASM_AX), %r13 - mov VCPU_R14(%_ASM_AX), %r14 - mov VCPU_R15(%_ASM_AX), %r15 + mov VCPU_R8 (%_ASM_DI), %r8 + mov VCPU_R9 (%_ASM_DI), %r9 + mov VCPU_R10(%_ASM_DI), %r10 + mov VCPU_R11(%_ASM_DI), %r11 + mov VCPU_R12(%_ASM_DI), %r12 + mov VCPU_R13(%_ASM_DI), %r13 + mov VCPU_R14(%_ASM_DI), %r14 + mov VCPU_R15(%_ASM_DI), %r15 #endif - /* Load guest RAX. This kills the @vmx pointer! */ - mov VCPU_RAX(%_ASM_AX), %_ASM_AX + /* Load guest RDI. This kills the @vmx pointer! */ + mov VCPU_RDI(%_ASM_DI), %_ASM_DI /* * Note, ALTERNATIVE_2 works in reverse order. If CLEAR_CPU_BUF_VM is
@@ -207,29 +204,29 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) UNWIND_HINT_RESTORE ENDBR - /* Temporarily save guest's RAX. */ - push %_ASM_AX + /* Temporarily save guest's RDI. */ + push %_ASM_DI - /* Reload @vmx to RAX. */ - mov 2*WORD_SIZE(%_ASM_SP), %_ASM_AX + /* Reload @vmx to RDI. */ + mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI - /* Save all guest registers, including RAX from the stack */ - pop VCPU_RAX(%_ASM_AX) - mov %_ASM_CX, VCPU_RCX(%_ASM_AX) - mov %_ASM_DX, VCPU_RDX(%_ASM_AX) - mov %_ASM_BX, VCPU_RBX(%_ASM_AX) - mov %_ASM_BP, VCPU_RBP(%_ASM_AX) - mov %_ASM_SI, VCPU_RSI(%_ASM_AX) - mov %_ASM_DI, VCPU_RDI(%_ASM_AX) + /* Save all guest registers, including RDI from the stack */ + mov %_ASM_AX, VCPU_RAX(%_ASM_DI) + mov %_ASM_CX, VCPU_RCX(%_ASM_DI) + mov %_ASM_DX, VCPU_RDX(%_ASM_DI) + mov %_ASM_BX, VCPU_RBX(%_ASM_DI) + mov %_ASM_BP, VCPU_RBP(%_ASM_DI) + mov %_ASM_SI, VCPU_RSI(%_ASM_DI) + pop VCPU_RDI(%_ASM_DI) #ifdef CONFIG_X86_64 - mov %r8, VCPU_R8 (%_ASM_AX) - mov %r9, VCPU_R9 (%_ASM_AX) - mov %r10, VCPU_R10(%_ASM_AX) - mov %r11, VCPU_R11(%_ASM_AX) - mov %r12, VCPU_R12(%_ASM_AX) - mov %r13, VCPU_R13(%_ASM_AX) - mov %r14, VCPU_R14(%_ASM_AX) - mov %r15, VCPU_R15(%_ASM_AX) + mov %r8, VCPU_R8 (%_ASM_DI) + mov %r9, VCPU_R9 (%_ASM_DI) + mov %r10, VCPU_R10(%_ASM_DI) + mov %r11, VCPU_R11(%_ASM_DI) + mov %r12, VCPU_R12(%_ASM_DI) + mov %r13, VCPU_R13(%_ASM_DI) + mov %r14, VCPU_R14(%_ASM_DI) + mov %r15, VCPU_R15(%_ASM_DI) #endif /* Clear return value to indicate VM-Exit (as opposed to VM-Fail). */
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 3/8 KVM: SVM: prepare for making SPEC_CTRL switch common with VMX
In reply to Paolo BonziniRemove the local labels and restrict RESTORE_*_BODY to just the restore code itself. Since the alternatives are different between VMX and SVM, having labels in the per-vendor file and jumps in another would be too confusing.
--- arch/x86/kvm/svm/vmenter.S | 54 +++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 24 deletions(-)
arch/x86/kvm/svm/vmenter.S +30 -24
@@ -39,10 +39,8 @@ ALTERNATIVE_2 "", \ "jmp 800f", X86_FEATURE_MSR_SPEC_CTRL, \ "", X86_FEATURE_V_SPEC_CTRL -801: .endm -.macro RESTORE_GUEST_SPEC_CTRL_BODY -800: +.macro RESTORE_GUEST_SPEC_CTRL_BODY guest_spec_ctrl:req, label:req /* * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the * host's, write the MSR. This is kept out-of-line so that the common
@@ -53,24 +51,23 @@ * and vmentry. */ #ifdef CONFIG_X86_64 - mov SVM_spec_ctrl(%rdi), %rdx + mov \guest_spec_ctrl, %rdx cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx - je 801b + je \label movl %edx, %eax shr $32, %rdx #else - mov SVM_spec_ctrl(%edi), %eax + mov \guest_spec_ctrl, %eax mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx xor %eax, %ecx - mov SVM_spec_ctrl + 4(%edi), %edx + mov 4 + \guest_spec_ctrl, %edx mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi xor %edx, %esi or %esi, %ecx - je 801b + je \label #endif mov $MSR_IA32_SPEC_CTRL, %ecx wrmsr - jmp 801b .endm .macro RESTORE_HOST_SPEC_CTRL
@@ -78,10 +75,8 @@ ALTERNATIVE_2 "", \ "jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \ "", X86_FEATURE_V_SPEC_CTRL -901: .endm -.macro RESTORE_HOST_SPEC_CTRL_BODY spec_ctrl_intercepted:req -900: +.macro RESTORE_HOST_SPEC_CTRL_BODY guest_spec_ctrl:req, spec_ctrl_intercepted:req, label:req /* Same for after vmexit. */ mov $MSR_IA32_SPEC_CTRL, %ecx
@@ -92,28 +87,27 @@ cmpb $0, \spec_ctrl_intercepted jnz 998f rdmsr - movl %eax, SVM_spec_ctrl(%_ASM_DI) - movl %edx, SVM_spec_ctrl + 4(%_ASM_DI) + movl %eax, \guest_spec_ctrl + movl %edx, 4 + \guest_spec_ctrl 998: /* Now restore the host value of the MSR if different from the guest's. */ #ifdef CONFIG_X86_64 mov PER_CPU_VAR(x86_spec_ctrl_current), %rdx - cmp SVM_spec_ctrl(%rdi), %rdx - je 901b + cmp \guest_spec_ctrl, %rdx + je \label movl %edx, %eax shr $32, %rdx #else mov PER_CPU_VAR(x86_spec_ctrl_current), %eax - mov SVM_spec_ctrl(%edi), %esi + mov \guest_spec_ctrl, %esi xor %eax, %esi mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edx - mov SVM_spec_ctrl + 4(%edi), %edi + mov 4 + \guest_spec_ctrl, %edi xor %edx, %edi or %edi, %esi - je 901b + je \label #endif wrmsr - jmp 901b .endm #define SVM_CLEAR_CPU_BUFFERS \
@@ -162,6 +156,7 @@ SYM_FUNC_START(__svm_vcpu_run) /* Clobbers RAX, RCX, RDX (and ESI on 32-bit), consumes RDI (@svm). */ RESTORE_GUEST_SPEC_CTRL +801: /* * Use a single vmcb (vmcb01 because it's always valid) for
@@ -242,6 +237,7 @@ SYM_FUNC_START(__svm_vcpu_run) * and RSP (pointer to @spec_ctrl_intercepted). */ RESTORE_HOST_SPEC_CTRL +901: /* * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
@@ -295,8 +291,12 @@ SYM_FUNC_START(__svm_vcpu_run) pop %_ASM_BP RET - RESTORE_GUEST_SPEC_CTRL_BODY - RESTORE_HOST_SPEC_CTRL_BODY (%_ASM_SP) +800: + RESTORE_GUEST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), 801b + jmp 801b +900: + RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), (%_ASM_SP), 901b + jmp 901b 10: cmpb $0, _ASM_RIP(virt_rebooting) jne 2b
@@ -362,6 +362,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* Clobbers RAX, RCX, and RDX (@hostsa), consumes RDI (@svm). */ RESTORE_GUEST_SPEC_CTRL +801: /* Get svm->current_vmcb->pa into RAX. */ mov SVM_current_vmcb(%rdi), %rax
@@ -378,6 +379,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* Clobbers RAX, RCX, RDX, consumes RDI (@svm) and RSI (@spec_ctrl_intercepted). */ RESTORE_HOST_SPEC_CTRL +901: /* * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
@@ -391,8 +393,12 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) FRAME_END RET - RESTORE_GUEST_SPEC_CTRL_BODY - RESTORE_HOST_SPEC_CTRL_BODY %sil +800: + RESTORE_GUEST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), 801b + jmp 801b +900: + RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), %sil, 901b + jmp 901b 3: cmpb $0, virt_rebooting(%rip) jne 2b
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 4/8 KVM: SVM: pass struct vcpu_svm to msr_write_intercepted
In reply to Paolo BonziniRemove an unnecessary difference between Intel and AMD.
--- arch/x86/kvm/svm/svm.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-)
arch/x86/kvm/svm/svm.c +5 -6
@@ -661,7 +661,7 @@ static void clr_dr_intercepts(struct vcpu_svm *svm)svm_mark_intercepts_dirty (svm); } -static bool msr_write_intercepted (struct kvm_vcpu *vcpu,u32 msr) +static bool msr_write_intercepted (struct vcpu_svm *svm,u32 msr) {/* * For non-nested case:
@@ -672,8 +672,7 @@ static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)* If the L02 MSR bitmap does not intercept the MSR, then we need to * save it. */ -void *msrpm =is_guest_mode (vcpu) ?to_svm (vcpu)->nested.msrpm : -to_svm (vcpu)->msrpm; +void *msrpm =is_guest_mode (&svm->vcpu) ? svm->nested.msrpm : svm->msrpm;return svm_test_msr_bitmap_write (msrpm, msr); }
@@ -2764,7 +2763,7 @@ static bool sev_es_prevent_msr_access(struct kvm_vcpu *vcpu, {return is_sev_es_guest (vcpu) && vcpu->arch.guest_state_protected && msr_info->index !=MSR_IA32_XSS && - !msr_write_intercepted (vcpu, msr_info->index); + !msr_write_intercepted (to_svm (vcpu), msr_info->index); }static int svm_get_msr (struct kvm_vcpu *vcpu,struct msr_data *msr_info)
@@ -4414,7 +4413,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) {bool force_immediate_exit = run_flags &KVM_RUN_FORCE_IMMEDIATE_EXIT ;struct vcpu_svm *svm =to_svm (vcpu); -bool spec_ctrl_intercepted =msr_write_intercepted (vcpu,MSR_IA32_SPEC_CTRL ); +bool spec_ctrl_intercepted =msr_write_intercepted (svm,MSR_IA32_SPEC_CTRL );trace_kvm_entry (vcpu, force_immediate_exit);
@@ -4556,7 +4555,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET ; -if (!msr_write_intercepted (vcpu,MSR_AMD64_PERF_CNTR_GLOBAL_CTL )) +if (!msr_write_intercepted (svm,MSR_AMD64_PERF_CNTR_GLOBAL_CTL ))rdmsrq (MSR_AMD64_PERF_CNTR_GLOBAL_CTL ,vcpu_to_pmu (vcpu)->global_ctrl);trace_kvm_exit (vcpu,KVM_ISA_SVM );
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 5/8 KVM: SVM: adopt the same VMX_RUN_* flags as VMX
In reply to Paolo BonziniRename VMX_RUN_* to KVM_ENTER_* (to not confuse with KVM_RUN_* that already exists) and replace SVM's spec_ctrl_intercepted with that same bitmask.
Note that KVM_ENTER_SAVE_SPEC_CTRL has the opposite polarity compared to spec_ctrl_intercepted. That is, the guest value of the MSR must be saved if it writes are *not* intercepted.
--- arch/x86/kvm/svm/svm.c | 14 +++++++++----- arch/x86/kvm/svm/svm.h | 4 ++-- arch/x86/kvm/svm/vmenter.S | 23 ++++++++++++----------- arch/x86/kvm/vmenter.h | 9 +++++++++ arch/x86/kvm/vmx/run_flags.h | 9 --------- arch/x86/kvm/vmx/vmenter.S | 12 ++++++------ arch/x86/kvm/vmx/vmx.c | 13 +++++++------ arch/x86/kvm/vmx/vmx.h | 3 +-- 8 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 arch/x86/kvm/vmenter.h delete mode 100644 arch/x86/kvm/vmx/run_flags.h
arch/x86/kvm/svm/svm.c +9 -5
@@ -50,6 +50,7 @@#include "trace.h" +#include "vmenter.h" #include "svm.h" #include "svm_ops.h"
@@ -4377,7 +4378,7 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)return EXIT_FASTPATH_NONE ; } -static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted) +static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned enter_flags) {struct svm_cpu_data *sd =per_cpu_ptr (&svm_data, vcpu->cpu);struct vcpu_svm *svm =to_svm (vcpu);
@@ -4399,10 +4400,10 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_inamd_clear_divider ();if (is_sev_es_guest (vcpu)) -__svm_sev_es_vcpu_run (svm, spec_ctrl_intercepted, +__svm_sev_es_vcpu_run (svm, enter_flags,sev_es_host_save_area (sd));else -__svm_vcpu_run (svm, spec_ctrl_intercepted); +__svm_vcpu_run (svm, enter_flags);raw_local_irq_disable ();
@@ -4413,7 +4414,10 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) {bool force_immediate_exit = run_flags &KVM_RUN_FORCE_IMMEDIATE_EXIT ;struct vcpu_svm *svm =to_svm (vcpu); -bool spec_ctrl_intercepted =msr_write_intercepted (svm,MSR_IA32_SPEC_CTRL ); +unsigned enter_flags =0 ; + +if (!msr_write_intercepted (svm,MSR_IA32_SPEC_CTRL )) + enter_flags |=KVM_ENTER_SAVE_SPEC_CTRL ;trace_kvm_entry (vcpu, force_immediate_exit);
@@ -4496,7 +4500,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)if (!static_cpu_has (X86_FEATURE_V_SPEC_CTRL ))x86_spec_ctrl_set_guest (svm->virt_spec_ctrl); -svm_vcpu_enter_exit (vcpu, spec_ctrl_intercepted); +svm_vcpu_enter_exit (vcpu, enter_flags);if (!static_cpu_has (X86_FEATURE_V_SPEC_CTRL ))x86_spec_ctrl_restore_host (svm->virt_spec_ctrl);
arch/x86/kvm/svm/svm.h +2 -2
@@ -1007,9 +1007,9 @@ static inline void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_sa/* vmenter.S */ -void __svm_sev_es_vcpu_run (struct vcpu_svm *svm,bool spec_ctrl_intercepted, +void __svm_sev_es_vcpu_run (struct vcpu_svm *svm,unsigned int flags,struct sev_es_save_area *hostsa); -void __svm_vcpu_run (struct vcpu_svm *svm,bool spec_ctrl_intercepted); +void __svm_vcpu_run (struct vcpu_svm *svm,unsigned int flags);#define DEFINE_KVM_GHCB_ACCESSORS (field) \ static __always_inline u64 kvm_ghcb_get_##field(struct vcpu_svm *svm) \
arch/x86/kvm/svm/vmenter.S +12 -11
@@ -7,6 +7,7 @@ #include <asm/kvm_vcpu_regs.h> #include <asm/nospec-branch.h> #include "kvm-asm-offsets.h" +#include "vmenter.h" #define WORD_SIZE (BITS_PER_LONG / 8)
@@ -76,7 +77,7 @@ "jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \ "", X86_FEATURE_V_SPEC_CTRL .endm -.macro RESTORE_HOST_SPEC_CTRL_BODY guest_spec_ctrl:req, spec_ctrl_intercepted:req, label:req +.macro RESTORE_HOST_SPEC_CTRL_BODY guest_spec_ctrl:req, enter_flags:req, label:req /* Same for after vmexit. */ mov $MSR_IA32_SPEC_CTRL, %ecx
@@ -84,8 +85,8 @@ * Load the value that the guest had written into MSR_IA32_SPEC_CTRL, * if it was not intercepted during guest execution. */ - cmpb $0, \spec_ctrl_intercepted - jnz 998f + testl $KVM_ENTER_SAVE_SPEC_CTRL, \enter_flags + jz 998f rdmsr movl %eax, \guest_spec_ctrl movl %edx, 4 + \guest_spec_ctrl
@@ -115,8 +116,8 @@ /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode - * @svm: struct vcpu_svm * - * @spec_ctrl_intercepted: bool + * @svm: struct vcpu_svm * + * @enter_flags: u32 */ SYM_FUNC_START(__svm_vcpu_run) push %_ASM_BP
@@ -274,7 +275,7 @@ SYM_FUNC_START(__svm_vcpu_run) xor %r15d, %r15d #endif - /* "Pop" @spec_ctrl_intercepted. */ + /* "Pop" @enter_flags. */ pop %_ASM_BX pop %_ASM_BX
@@ -335,8 +336,8 @@ SYM_FUNC_END(__svm_vcpu_run) /** * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode - * @svm: struct vcpu_svm * - * @spec_ctrl_intercepted: bool + * @svm: struct vcpu_svm * + * @enter_flags: u32 */ SYM_FUNC_START(__svm_sev_es_vcpu_run) FRAME_BEGIN
@@ -355,7 +356,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* * Save volatile registers that hold arguments that are needed after - * #VMEXIT (RDI=@svm and RSI=@spec_ctrl_intercepted). + * #VMEXIT (RDI=@svm and RSI=@enter_flags). */ mov %rdi, SEV_ES_RDI (%rdx) mov %rsi, SEV_ES_RSI (%rdx)
@@ -377,7 +378,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */ FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT - /* Clobbers RAX, RCX, RDX, consumes RDI (@svm) and RSI (@spec_ctrl_intercepted). */ + /* Clobbers RAX, RCX, RDX, consumes RDI (@svm) and RSI (@enter_flags). */ RESTORE_HOST_SPEC_CTRL 901:
@@ -397,7 +398,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) RESTORE_GUEST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), 801b jmp 801b 900: - RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), %sil, 901b + RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), %esi, 901b jmp 901b 3: cmpb $0, virt_rebooting(%rip)
arch/x86/kvm/vmenter.h new file +9 -0
@@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __KVM_X86_VMENTER_H +#define __KVM_X86_VMENTER_H + +#define KVM_ENTER_VMRESUME BIT(0) +#define KVM_ENTER_SAVE_SPEC_CTRL BIT(1) +#define KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO BIT(2) + +#endif /* __KVM_X86_VMENTER_H */
arch/x86/kvm/vmx/run_flags.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __KVM_X86_VMX_RUN_FLAGS_H -#define __KVM_X86_VMX_RUN_FLAGS_H - -#define VMX_RUN_VMRESUME BIT(0) -#define VMX_RUN_SAVE_SPEC_CTRL BIT(1) -#define VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO BIT(2) - -#endif /* __KVM_X86_VMX_RUN_FLAGS_H */
arch/x86/kvm/vmx/vmenter.S +6 -6
@@ -7,7 +7,7 @@ #include <asm/percpu.h> #include <asm/segment.h> #include "kvm-asm-offsets.h" -#include "run_flags.h" +#include "vmenter.h" #define WORD_SIZE (BITS_PER_LONG / 8)
@@ -68,9 +68,9 @@ /** * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode * @vmx: struct vcpu_vmx * - * @flags: VMX_RUN_VMRESUME: use VMRESUME instead of VMLAUNCH - * VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl - * VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO: vCPU can access host MMIO + * @flags: KVM_ENTER_VMRESUME: use VMRESUME instead of VMLAUNCH + * KVM_ENTER_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl + * KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO: vCPU can access host MMIO * * Returns: * 0 on VM-Exit, 1 on VM-Fail
@@ -164,7 +164,7 @@ SYM_FUNC_START(__vmx_vcpu_run) * do VERW. Else, do nothing (no mitigations needed/enabled). */ ALTERNATIVE_2 "", \ - __stringify(testl $VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO, (%_ASM_SP); \ + __stringify(testl $KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO, (%_ASM_SP); \ jz .Lskip_mmio_verw; \ VERW; \ .Lskip_mmio_verw:), \
@@ -172,7 +172,7 @@ SYM_FUNC_START(__vmx_vcpu_run) __stringify(VERW), X86_FEATURE_CLEAR_CPU_BUF_VM /* Check @flags to see if VMLAUNCH or VMRESUME is needed. */ - testl $VMX_RUN_VMRESUME, (%_ASM_SP) + testl $KVM_ENTER_VMRESUME, (%_ASM_SP) jz .Lvmlaunch /*
arch/x86/kvm/vmx/vmx.c +7 -6
@@ -73,6 +73,7 @@#include "x86_ops.h" #include "smm.h" #include "vmx_onhyperv.h" +#include "vmenter.h" #include "posted_intr.h" #include "mmu/spte.h"
@@ -964,12 +965,12 @@ static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr)return vmx_test_msr_bitmap_write (vmx->loaded_vmcs->msr_bitmap, msr); } -unsigned int __vmx_vcpu_run_flags (struct vcpu_vmx *vmx) +unsigned int __vmx_vcpu_enter_flags (struct vcpu_vmx *vmx) {unsigned int flags =0 ;if (vmx->loaded_vmcs->launched) - flags |=VMX_RUN_VMRESUME ; + flags |=KVM_ENTER_VMRESUME ;/* * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free
@@ -977,11 +978,11 @@ unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx)* it after vmexit and store it in vmx->spec_ctrl. */ if (!msr_write_intercepted (vmx,MSR_IA32_SPEC_CTRL )) - flags |=VMX_RUN_SAVE_SPEC_CTRL ; + flags |=KVM_ENTER_SAVE_SPEC_CTRL ;if (cpu_feature_enabled (X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO ) &&kvm_vcpu_can_access_host_mmio (&vmx->vcpu)) - flags |=VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO ; + flags |=KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO ;return flags; }
@@ -7395,7 +7396,7 @@ void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx,if (!cpu_feature_enabled (X86_FEATURE_MSR_SPEC_CTRL ))return ; -if (flags &VMX_RUN_SAVE_SPEC_CTRL ) +if (flags &KVM_ENTER_SAVE_SPEC_CTRL ) vmx->spec_ctrl =native_rdmsrq (MSR_IA32_SPEC_CTRL );/*
@@ -7586,7 +7587,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)kvm_wait_lapic_expire (vcpu);/* The actual VMENTER/EXIT is in the .noinstr.text section. */ -vmx_vcpu_enter_exit (vcpu,__vmx_vcpu_run_flags (vmx)); +vmx_vcpu_enter_exit (vcpu,__vmx_vcpu_enter_flags (vmx));/* All fields are clean at this point */ if (kvm_is_using_evmcs ()) {
arch/x86/kvm/vmx/vmx.h +1 -2
@@ -15,7 +15,6 @@#include "vmcs.h" #include "vmx_ops.h" #include "../cpuid.h" -#include "run_flags.h" #include "../mmu.h" #include "common.h"
@@ -369,7 +368,7 @@ struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr);void pt_update_intercept_for_msr (struct kvm_vcpu *vcpu);void vmx_update_host_rsp (struct vcpu_vmx *vmx,unsigned long host_rsp);void vmx_spec_ctrl_restore_host (struct vcpu_vmx *vmx,unsigned int flags); -unsigned int __vmx_vcpu_run_flags (struct vcpu_vmx *vmx); +unsigned int __vmx_vcpu_enter_flags (struct vcpu_vmx *vmx);bool __vmx_vcpu_run (struct vcpu_vmx *vmx,unsigned int flags);void vmx_ept_load_pdptrs (struct kvm_vcpu *vcpu);
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 6/8 KVM: SVM: extract RESTORE_*_SPEC_CTRL_BODY out of svm/vmenter.S
In reply to Paolo BonziniPure code movement, in preparation for using these macros for VMX as well.
--- arch/x86/kvm/svm/vmenter.S | 62 ----------------------------------- arch/x86/kvm/vmenter.h | 66 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 62 deletions(-)
arch/x86/kvm/svm/vmenter.S +0 -62
@@ -41,35 +41,6 @@ "jmp 800f", X86_FEATURE_MSR_SPEC_CTRL, \ "", X86_FEATURE_V_SPEC_CTRL .endm -.macro RESTORE_GUEST_SPEC_CTRL_BODY guest_spec_ctrl:req, label:req - /* - * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the - * host's, write the MSR. This is kept out-of-line so that the common - * case does not have to jump. - * - * IMPORTANT: To avoid RSB underflow attacks and any other nastiness, - * there must not be any returns or indirect branches between this code - * and vmentry. - */ -#ifdef CONFIG_X86_64 - mov \guest_spec_ctrl, %rdx - cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx - je \label - movl %edx, %eax - shr $32, %rdx -#else - mov \guest_spec_ctrl, %eax - mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx - xor %eax, %ecx - mov 4 + \guest_spec_ctrl, %edx - mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi - xor %edx, %esi - or %esi, %ecx - je \label -#endif - mov $MSR_IA32_SPEC_CTRL, %ecx - wrmsr -.endm .macro RESTORE_HOST_SPEC_CTRL /* No need to do anything if SPEC_CTRL is unset or V_SPEC_CTRL is set */
@@ -77,39 +48,6 @@ "jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \ "", X86_FEATURE_V_SPEC_CTRL .endm -.macro RESTORE_HOST_SPEC_CTRL_BODY guest_spec_ctrl:req, enter_flags:req, label:req - /* Same for after vmexit. */ - mov $MSR_IA32_SPEC_CTRL, %ecx - - /* - * Load the value that the guest had written into MSR_IA32_SPEC_CTRL, - * if it was not intercepted during guest execution. - */ - testl $KVM_ENTER_SAVE_SPEC_CTRL, \enter_flags - jz 998f - rdmsr - movl %eax, \guest_spec_ctrl - movl %edx, 4 + \guest_spec_ctrl -998: - /* Now restore the host value of the MSR if different from the guest's. */ -#ifdef CONFIG_X86_64 - mov PER_CPU_VAR(x86_spec_ctrl_current), %rdx - cmp \guest_spec_ctrl, %rdx - je \label - movl %edx, %eax - shr $32, %rdx -#else - mov PER_CPU_VAR(x86_spec_ctrl_current), %eax - mov \guest_spec_ctrl, %esi - xor %eax, %esi - mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edx - mov 4 + \guest_spec_ctrl, %edi - xor %edx, %edi - or %edi, %esi - je \label -#endif - wrmsr -.endm #define SVM_CLEAR_CPU_BUFFERS \ ALTERNATIVE "", __CLEAR_CPU_BUFFERS, X86_FEATURE_CLEAR_CPU_BUF_VM
arch/x86/kvm/vmenter.h +66 -0
70 lines not on this page · raw message
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 7/8 KVM: VMX: switch to RESTORE_GUEST_SPEC_CTRL_BODY
In reply to Paolo BonziniThis has exactly the same expansion, so there is no change.
--- arch/x86/kvm/vmx/vmenter.S | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-)
arch/x86/kvm/vmx/vmenter.S +4 -27
@@ -101,35 +101,12 @@ SYM_FUNC_START(__vmx_vcpu_run) /* Reload @vmx, _ASM_ARG1 may be modified by vmx_update_host_rsp(). */ mov WORD_SIZE(%_ASM_SP), %_ASM_DI - ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL - /* - * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the - * host's, write the MSR. - * - * IMPORTANT: To avoid RSB underflow attacks and any other nastiness, - * there must not be any returns or indirect branches between this code - * and vmentry. + * Unlike AMD there's no V_SPEC_CTRL here, so do not leave the body + * out of line. Clobbers RAX, RCX, RDX, RSI. */ -#ifdef CONFIG_X86_64 - mov VMX_spec_ctrl(%rdi), %rdx - cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx - je .Lspec_ctrl_done - movl %edx, %eax - shr $32, %rdx -#else - mov VMX_spec_ctrl(%edi), %eax - mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx - xor %eax, %ecx - mov VMX_spec_ctrl + 4(%edi), %edx - mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi - xor %edx, %esi - or %esi, %ecx - je .Lspec_ctrl_done -#endif - mov $MSR_IA32_SPEC_CTRL, %ecx - wrmsr - + ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL + RESTORE_GUEST_SPEC_CTRL_BODY VMX_spec_ctrl(%_ASM_DI), .Lspec_ctrl_done .Lspec_ctrl_done: /*
Signature · 1 line
2.52.0
Paolo Bonzini
maintainerPATCH v3 8/8 KVM: VMX: replace vmx_spec_ctrl_restore_host with RESTORE_HOST_SPEC_CTRL_BODY
In reply to Paolo BonziniReuse the same assembly as SVM, just with alternatives instead of cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS).
Note that Intel does need an LFENCE with eIBRS, unlike AMD's AutoIBRS. However, it is not needed for X86_FEATURE_KERNEL_IBRS because there are no conditional branches between FILL_RETURN_BUFFER and ret.
--- arch/x86/kvm/vmenter.h | 9 +++++++-- arch/x86/kvm/vmx/vmenter.S | 28 ++++++++++++++++++++++------ arch/x86/kvm/vmx/vmx.c | 25 ------------------------- arch/x86/kvm/vmx/vmx.h | 1 - 4 files changed, 29 insertions(+), 34 deletions(-)
arch/x86/kvm/vmenter.h +7 -2
@@ -55,7 +55,12 @@#ifdef CONFIG_X86_64 mov PER_CPU_VAR(x86_spec_ctrl_current), %rdx cmp \guest_spec_ctrl, %rdx - je \label + /* + * For legacy IBRS, the IBRS bit always needs to be written after + * transitioning from a less privileged predictor mode, regardless of + * whether the guest/host values differ. + */ + ALTERNATIVE __stringify(je \label), "", X86_FEATURE_KERNEL_IBRS movl %edx, %eax shr $32, %rdx #else
@@ -66,7 +71,7 @@ mov 4 + \guest_spec_ctrl, %edi xor %edx, %edi or %edi, %esi - je \label + ALTERNATIVE __stringify(je \label), "", X86_FEATURE_KERNEL_IBRS#endif wrmsr .endm
arch/x86/kvm/vmx/vmenter.S +22 -6
@@ -105,9 +105,9 @@ SYM_FUNC_START(__vmx_vcpu_run) * Unlike AMD there's no V_SPEC_CTRL here, so do not leave the body * out of line. Clobbers RAX, RCX, RDX, RSI. */ - ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL - RESTORE_GUEST_SPEC_CTRL_BODY VMX_spec_ctrl(%_ASM_DI), .Lspec_ctrl_done -.Lspec_ctrl_done: + ALTERNATIVE "jmp .Lspec_ctrl_guest_done", "", X86_FEATURE_MSR_SPEC_CTRL + RESTORE_GUEST_SPEC_CTRL_BODY VMX_spec_ctrl(%_ASM_DI), .Lspec_ctrl_guest_done +.Lspec_ctrl_guest_done: /* * Since vmentry is serializing on affected CPUs, there's no need for
@@ -252,16 +252,32 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT,\ X86_FEATURE_RSB_VMEXIT_LITE - pop %_ASM_ARG2 /* @flags */ - pop %_ASM_ARG1 /* @vmx */ + /* Clobbers RAX, RCX, RDX, RSI. */ + ALTERNATIVE "jmp .Lspec_ctrl_host_done", "", X86_FEATURE_MSR_SPEC_CTRL + mov WORD_SIZE(%_ASM_SP), %_ASM_DI + RESTORE_HOST_SPEC_CTRL_BODY VMX_spec_ctrl(%_ASM_DI), (%_ASM_SP), .Lspec_ctrl_host_done +.Lspec_ctrl_host_done: - call vmx_spec_ctrl_restore_host + /* + * Halt speculation past a conditional wrmsr. Intel's eIBRS + * guarantees that the guest cannot control the RSB "once IBRS is + * set", but in the eIBRS case speculative execution past the 'je' + * can go all the way to the RET below while MSR_IA32_SPEC_CTRL + * still holds the guest value. + */ + ALTERNATIVE_2 "", "lfence", X86_FEATURE_MSR_SPEC_CTRL, \ + "", X86_FEATURE_KERNEL_IBRS CLEAR_BRANCH_HISTORY_VMEXIT /* Put return value in AX */ mov %_ASM_BX, %_ASM_AX + /* Pop our saved arguments from the stack */ + pop %_ASM_BX + pop %_ASM_BX + + /* ... and then the callee-save registers */ pop %_ASM_BX #ifdef CONFIG_X86_64 pop %r12
arch/x86/kvm/vmx/vmx.c +0 -25
@@ -7388,31 +7388,6 @@ void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) } } -void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, -unsigned int flags) -{ -u64 hostval =this_cpu_read (x86_spec_ctrl_current); - -if (!cpu_feature_enabled (X86_FEATURE_MSR_SPEC_CTRL )) -return ; - -if (flags &KVM_ENTER_SAVE_SPEC_CTRL ) - vmx->spec_ctrl =native_rdmsrq (MSR_IA32_SPEC_CTRL ); - -/* -* If the guest/host SPEC_CTRL values differ, restore the host value. -* -* For legacy IBRS, the IBRS bit always needs to be written after -* transitioning from a less privileged predictor mode, regardless of -* whether the guest/host values differ. -*/ -if (cpu_feature_enabled (X86_FEATURE_KERNEL_IBRS ) || - vmx->spec_ctrl != hostval) -native_wrmsrq (MSR_IA32_SPEC_CTRL , hostval); - -barrier_nospec (); -} -static fastpath_t vmx_exit_handlers_fastpath (struct kvm_vcpu *vcpu,bool force_immediate_exit) {
arch/x86/kvm/vmx/vmx.h +0 -1
@@ -367,7 +367,6 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu);struct vmx_uret_msr *vmx_find_uret_msr (struct vcpu_vmx *vmx,u32 msr);void pt_update_intercept_for_msr (struct kvm_vcpu *vcpu);void vmx_update_host_rsp (struct vcpu_vmx *vmx,unsigned long host_rsp); -void vmx_spec_ctrl_restore_host (struct vcpu_vmx *vmx,unsigned int flags);unsigned int __vmx_vcpu_enter_flags (struct vcpu_vmx *vmx);bool __vmx_vcpu_run (struct vcpu_vmx *vmx,unsigned int flags);void vmx_ept_load_pdptrs (struct kvm_vcpu *vcpu);
Signature · 1 line
2.52.0