KVM: arm64: ARMv8.3/8.4 Nested Virtualization support
Subsystems (4)
Marc Zyngier
maintainerPATCH v5 49/69 KVM: arm64: nv: Add handling of EL2-specific timer registers
In reply to Marc ZyngierAdd the required handling for EL2 and EL02 registers, as well as EL1 registers used in the E2H context.
--- arch/arm64/include/asm/sysreg.h | 6 +++ arch/arm64/kvm/sys_regs.c | 87 +++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+)
arch/arm64/include/asm/sysreg.h +6 -0
@@ -625,6 +625,12 @@#define SYS_CNTVOFF_EL2 sys_reg(3, 4, 14, 0, 3)#define SYS_CNTHCTL_EL2 sys_reg(3, 4, 14, 1, 0) +#define SYS_CNTHP_TVAL_EL2 sys_reg(3, 4, 14, 2, 0) +#define SYS_CNTHP_CTL_EL2 sys_reg(3, 4, 14, 2, 1) +#define SYS_CNTHP_CVAL_EL2 sys_reg(3, 4, 14, 2, 2) +#define SYS_CNTHV_TVAL_EL2 sys_reg(3, 4, 14, 3, 0) +#define SYS_CNTHV_CTL_EL2 sys_reg(3, 4, 14, 3, 1) +#define SYS_CNTHV_CVAL_EL2 sys_reg(3, 4, 14, 3, 2)/* VHE encodings for architectural EL0/1 system registers */ #define SYS_SCTLR_EL12 sys_reg(3, 5, 1, 0, 0)
arch/arm64/kvm/sys_regs.c +87 -0
119 lines not on this page · raw message
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 50/69 KVM: arm64: nv: Load timer before the GIC
In reply to Marc ZyngierIn order for vgic_v3_load_nested to be able to observe which timer interrupts have the HW bit set for the current context, the timers must have been loaded in the new mode and the right timer mapped to their corresponding HW IRQs.
At the moment, we load the GIC first, meaning that timer interrupts injected to an L2 guest will never have the HW bit set (we see the old configuration).
Swapping the two loads solves this particular problem.
--- arch/arm64/kvm/arm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/kvm/arm.c +1 -1
@@ -414,8 +414,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->cpu = cpu; -kvm_vgic_load (vcpu);kvm_timer_vcpu_load (vcpu); +kvm_vgic_load (vcpu);if (has_vhe ())kvm_vcpu_load_sysregs_vhe (vcpu);kvm_arch_vcpu_load_fp (vcpu);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 51/69 KVM: arm64: nv: Nested GICv3 Support
In reply to Marc ZyngierFrom: Jintack Lim <[email protected]>
When entering a nested VM, we set up the hypervisor control interface based on what the guest hypervisor has set. Especially, we investigate each list register written by the guest hypervisor whether HW bit is set. If so, we translate hw irq number from the guest's point of view to the real hardware irq number if there is a mapping.
--- arch/arm64/include/asm/kvm_emulate.h | 8 +- arch/arm64/include/asm/kvm_host.h | 11 +- arch/arm64/include/asm/kvm_nested.h | 1 + arch/arm64/kvm/Makefile | 2 +- arch/arm64/kvm/arm.c | 13 +- arch/arm64/kvm/nested.c | 16 +++ arch/arm64/kvm/sys_regs.c | 179 ++++++++++++++++++++++++++- arch/arm64/kvm/vgic/vgic-v3-nested.c | 178 ++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v3.c | 26 ++++ arch/arm64/kvm/vgic/vgic.c | 27 ++++ include/kvm/arm_vgic.h | 18 +++ 11 files changed, 469 insertions(+), 10 deletions(-) create mode 100644 arch/arm64/kvm/vgic/vgic-v3-nested.c
arch/arm64/include/asm/kvm_emulate.h +7 -1
@@ -505,7 +505,13 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)static inline unsigned long kvm_vcpu_get_mpidr_aff (struct kvm_vcpu *vcpu) { -return vcpu_read_sys_reg (vcpu,MPIDR_EL1 ) &MPIDR_HWID_BITMASK ; +/* +* Use the in-memory view for MPIDR_EL1. It can't be changed by the +* guest, and is also accessed from the context of *another* vcpu, +* so anything using some other state (such as the NV state that is +* used by vcpu_read_sys_reg) will eventually go wrong. +*/ +return __vcpu_sys_reg (vcpu,MPIDR_EL1 ) &MPIDR_HWID_BITMASK ; }static inline void kvm_vcpu_set_be (struct kvm_vcpu *vcpu)
arch/arm64/include/asm/kvm_host.h +6 -5
@@ -42,11 +42,12 @@#define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) -#define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) -#define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) -#define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3) -#define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) -#define KVM_REQ_RELOAD_PMU KVM_ARCH_REQ(5) +#define KVM_REQ_IRQ_PENDING KVM_ARCH_REQ(1) +#define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) +#define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3) +#define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) +#define KVM_REQ_RELOAD_PMU KVM_ARCH_REQ(5) +#define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(6)#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET)
arch/arm64/include/asm/kvm_nested.h +1 -0
@@ -67,6 +67,7 @@ extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);extern struct kvm_s2_mmu *lookup_s2_mmu (struct kvm *kvm,u64 vttbr,u64 hcr);extern void kvm_vcpu_load_hw_mmu (struct kvm_vcpu *vcpu);extern void kvm_vcpu_put_hw_mmu (struct kvm_vcpu *vcpu); +extern void check_nested_vcpu_requests (struct kvm_vcpu *vcpu);struct kvm_s2_trans {phys_addr_t output;
arch/arm64/kvm/Makefile +1 -1
@@ -22,6 +22,6 @@ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ vgic/vgic-v3.o vgic/vgic-v4.o \ vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ - vgic/vgic-its.o vgic/vgic-debug.o + vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o
arch/arm64/kvm/arm.c +11 -2
@@ -715,6 +715,8 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu)if (kvm_check_request (KVM_REQ_RELOAD_PMU , vcpu))kvm_pmu_handle_pmcr (vcpu,__vcpu_sys_reg (vcpu,PMCR_EL0 )); + +check_nested_vcpu_requests (vcpu); } }
@@ -813,9 +815,16 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)if (!ret) ret =1 ; -update_vmid (&vcpu->arch.hw_mmu->vmid); - +/* +* A nested exeption triggered by a vcpu request (such +* as an interrupt injected in a guest hypervisor) can +* change the currently used VMID (by switching to a +* different translation regime. It is thus necesary +* to update the VMID *after* all requests have been +* processed. +*/ check_vcpu_requests (vcpu); +update_vmid (&vcpu->arch.hw_mmu->vmid);/* * Preparing the interrupts to be injected also
arch/arm64/kvm/nested.c +16 -0
@@ -592,6 +592,22 @@ void kvm_arch_flush_shadow_all(struct kvm *kvm)kvm_free_stage2_pgd (&kvm->arch.mmu); } +bool vgic_state_is_nested (struct kvm_vcpu *vcpu) +{ +bool imo =__vcpu_sys_reg (vcpu,HCR_EL2 ) &HCR_IMO ; +bool fmo =__vcpu_sys_reg (vcpu,HCR_EL2 ) &HCR_FMO ; + +WARN_ONCE (imo != fmo,"Separate virtual IRQ/FIQ settings not supported\n" ); + +return nested_virt_in_use (vcpu) && imo && fmo && !is_hyp_ctxt (vcpu); +} + +void check_nested_vcpu_requests (struct kvm_vcpu *vcpu) +{ +if (kvm_check_request (KVM_REQ_GUEST_HYP_IRQ_PENDING , vcpu)) +kvm_inject_nested_irq (vcpu); +} +/* * Our emulated CPU doesn't support all the possible features. For the * sake of simplicity (and probably mental sanity), wipe out a number
arch/arm64/kvm/sys_regs.c +178 -1
215 lines not on this page · raw message
arch/arm64/kvm/vgic/vgic-v3-nested.c new file +178 -0
178 lines not on this page · raw message
arch/arm64/kvm/vgic/vgic-v3.c +26 -0
@@ -8,6 +8,7 @@#include <kvm/arm_vgic.h> #include <asm/kvm_hyp.h> #include <asm/kvm_mmu.h> +#include <asm/kvm_nested.h> #include <asm/kvm_asm.h> #include "vgic.h"
@@ -277,6 +278,12 @@ void vgic_v3_enable(struct kvm_vcpu *vcpu) vgic_v3->vgic_sre = (ICC_SRE_EL1_DIB |ICC_SRE_EL1_DFB |ICC_SRE_EL1_SRE ); +/* +* If nesting is allowed, force GICv3 onto the nested +* guests as well. +*/ +if (nested_virt_in_use (vcpu)) + vcpu->arch.vgic_cpu.nested_vgic_v3.vgic_sre = vgic_v3->vgic_sre; vcpu->arch.vgic_cpu.pendbaser =INITIAL_PENDBASER_VALUE ; }else { vgic_v3->vgic_sre =0 ;
@@ -707,6 +714,13 @@ void vgic_v3_load(struct kvm_vcpu *vcpu) {struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; +/* +* vgic_v3_load_nested only affects the LRs in the shadow +* state, so it is fine to pass the nested state around. +*/ +if (vgic_state_is_nested (vcpu)) + cpu_if = &vcpu->arch.vgic_cpu.nested_vgic_v3; +/* * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
@@ -720,6 +734,9 @@ void vgic_v3_load(struct kvm_vcpu *vcpu)if (has_vhe ())__vgic_v3_activate_traps (cpu_if); +if (vgic_state_is_nested (vcpu)) +vgic_v3_load_nested (vcpu); +WARN_ON (vgic_v4_load (vcpu)); }
@@ -727,6 +744,9 @@ void vgic_v3_vmcr_sync(struct kvm_vcpu *vcpu) {struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; +if (vgic_state_is_nested (vcpu)) + cpu_if = &vcpu->arch.vgic_cpu.shadow_vgic_v3; +if (likely (cpu_if->vgic_sre)) cpu_if->vgic_vmcr =kvm_call_hyp_ret (__vgic_v3_read_vmcr); }
@@ -739,8 +759,14 @@ void vgic_v3_put(struct kvm_vcpu *vcpu)vgic_v3_vmcr_sync (vcpu); +if (vgic_state_is_nested (vcpu)) + cpu_if = &vcpu->arch.vgic_cpu.shadow_vgic_v3; +kvm_call_hyp (__vgic_v3_save_aprs, cpu_if);if (has_vhe ())__vgic_v3_deactivate_traps (cpu_if); + +if (vgic_state_is_nested (vcpu)) +vgic_v3_put_nested (vcpu); }
arch/arm64/kvm/vgic/vgic.c +27 -0
@@ -876,6 +876,10 @@ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) {int used_lrs; +/* If nesting, this is a load/put affair, not flush/sync. */ +if (vgic_state_is_nested (vcpu)) +return ; +/* An empty ap_list_head implies used_lrs == 0 */ if (list_empty (&vcpu->arch.vgic_cpu.ap_list_head))return ;
@@ -920,6 +924,29 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) !vgic_supports_direct_msis(vcpu->kvm))return ; +/* +* If in a nested state, we must return early. Two possibilities: +* +* - If we have any pending IRQ for the guest and the guest +* expects IRQs to be handled in its virtual EL2 mode (the +* virtual IMO bit is set) and it is not already running in +* virtual EL2 mode, then we have to emulate an IRQ +* exception to virtual EL2. +* +* We do that by placing a request to ourselves which will +* abort the entry procedure and inject the exception at the +* beginning of the run loop. +* +* - Otherwise, do exactly *NOTHING*. The guest state is +* already loaded, and we can carry on with running it. +*/ +if (vgic_state_is_nested (vcpu)) { +if (kvm_vgic_vcpu_pending_irq (vcpu)) +kvm_make_request (KVM_REQ_GUEST_HYP_IRQ_PENDING , vcpu); + +return ; + } +DEBUG_SPINLOCK_BUG_ON (!irqs_disabled ());if (!list_empty (&vcpu->arch.vgic_cpu.ap_list_head)) {
include/kvm/arm_vgic.h +18 -0
@@ -325,6 +325,15 @@ struct vgic_cpu {struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS ]; +/* CPU vif control registers for the virtual GICH interface */ +struct vgic_v3_cpu_if nested_vgic_v3; + +/* +* The shadow vif control register loaded to the hardware when +* running a nested L2 guest with the virtual IMO/FMO bit set. +*/ +struct vgic_v3_cpu_if shadow_vgic_v3; +raw_spinlock_t ap_list_lock;/* Protects the ap_list */ /*
@@ -382,6 +391,13 @@ void kvm_vgic_load(struct kvm_vcpu *vcpu);void kvm_vgic_put (struct kvm_vcpu *vcpu);void kvm_vgic_vmcr_sync (struct kvm_vcpu *vcpu); +void vgic_v3_load_nested (struct kvm_vcpu *vcpu); +void vgic_v3_put_nested (struct kvm_vcpu *vcpu); +void vgic_v3_handle_nested_maint_irq (struct kvm_vcpu *vcpu); +u16 vgic_v3_get_eisr (struct kvm_vcpu *vcpu); +u16 vgic_v3_get_elrsr (struct kvm_vcpu *vcpu); +u64 vgic_v3_get_misr (struct kvm_vcpu *vcpu); +#define irqchip_in_kernel (k) (!!((k)->arch.vgic.in_kernel))#define vgic_initialized (k) ((k)->arch.vgic.initialized)#define vgic_ready (k) ((k)->arch.vgic.ready)
@@ -426,4 +442,6 @@ int vgic_v4_load(struct kvm_vcpu *vcpu);void vgic_v4_commit (struct kvm_vcpu *vcpu);int vgic_v4_put (struct kvm_vcpu *vcpu,bool need_db); +bool vgic_state_is_nested (struct kvm_vcpu *vcpu); +#endif /* __KVM_ARM_VGIC_H */
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 52/69 KVM: arm64: nv: Don't load the GICv4 context on entering a nested guest
In reply to Marc ZyngierWhen entering a nested guest (vgic_state_is_nested() == true), special care must be taken *not* to make the vPE resident, as these are interrupts targetting the L1 guest, and not any nested guest.
By not making the vPE resident, we guarantee that the delivery of an vLPI will result in a doorbell, forcing an exit from the nested guest and a switch to the L1 guest to handle the interrupt.
--- arch/arm64/kvm/vgic/vgic-v3.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
arch/arm64/kvm/vgic/vgic-v3.c +8 -2
@@ -736,8 +736,8 @@ void vgic_v3_load(struct kvm_vcpu *vcpu)if (vgic_state_is_nested (vcpu))vgic_v3_load_nested (vcpu); - -WARN_ON (vgic_v4_load (vcpu)); +else +WARN_ON (vgic_v4_load (vcpu)); }void vgic_v3_vmcr_sync (struct kvm_vcpu *vcpu)
@@ -755,6 +755,12 @@ void vgic_v3_put(struct kvm_vcpu *vcpu) {struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; +/* +* vgic_v4_put will do nothing if we were not resident. This +* covers both the cases where we've blocked (we already have +* done a vgic_v4_put) and when running a nested guest (the +* vPE was never resident in order to generate a doorbell). +*/ WARN_ON (vgic_v4_put (vcpu, false));vgic_v3_vmcr_sync (vcpu);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 53/69 KVM: arm64: nv: vgic: Emulate the HW bit in software
In reply to Marc ZyngierFrom: Christoffer Dall <[email protected]>
Should the guest hypervisor use the HW bit in the LRs, we need to emulate the deactivation from the L2 guest into the L1 distributor emulation, which is handled by L0.
It's all good fun.
--- arch/arm64/include/asm/kvm_hyp.h | 2 ++ arch/arm64/kvm/hyp/vgic-v3-sr.c | 2 +- arch/arm64/kvm/vgic/vgic-v3-nested.c | 32 ++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.c | 6 ++++-- include/kvm/arm_vgic.h | 1 + 5 files changed, 40 insertions(+), 3 deletions(-)
arch/arm64/include/asm/kvm_hyp.h +2 -0
@@ -57,6 +57,8 @@ DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);int __vgic_v2_perform_cpuif_access (struct kvm_vcpu *vcpu); +u64 __gic_v3_get_lr (unsigned int lr); +void __vgic_v3_save_state (struct vgic_v3_cpu_if *cpu_if);void __vgic_v3_restore_state (struct vgic_v3_cpu_if *cpu_if);void __vgic_v3_activate_traps (struct vgic_v3_cpu_if *cpu_if);
arch/arm64/kvm/hyp/vgic-v3-sr.c +1 -1
@@ -18,7 +18,7 @@#define vtr_to_nr_pre_bits (v) ((((u32)(v) >> 26) & 7) + 1)#define vtr_to_nr_apr_regs (v) (1 << (vtr_to_nr_pre_bits(v) - 5)) -static u64 __gic_v3_get_lr (unsigned int lr) +u64 __gic_v3_get_lr (unsigned int lr) {switch (lr &0xf ) {case 0 :
arch/arm64/kvm/vgic/vgic-v3-nested.c +32 -0
@@ -138,6 +138,38 @@ static void vgic_v3_fixup_shadow_lr_state(struct kvm_vcpu *vcpu) } } +void vgic_v3_sync_nested (struct kvm_vcpu *vcpu) +{ +struct vgic_v3_cpu_if *cpu_if =vcpu_nested_if (vcpu); +struct vgic_v3_cpu_if *s_cpu_if =vcpu_shadow_if (vcpu); +struct vgic_irq *irq; +int i; + +for (i =0 ; i < s_cpu_if->used_lrs; i++) { +u64 lr = cpu_if->vgic_lr[i]; +int l1_irq; + +if (!(lr &ICH_LR_HW ) || !(lr &ICH_LR_STATE )) +continue ; + +/* +* If we had a HW lr programmed by the guest hypervisor, we +* need to emulate the HW effect between the guest hypervisor +* and the nested guest. +*/ + l1_irq = (lr &ICH_LR_PHYS_ID_MASK ) >>ICH_LR_PHYS_ID_SHIFT ; + irq =vgic_get_irq (vcpu->kvm, vcpu, l1_irq); +if (!irq) +continue ;/* oh well, the guest hyp is broken */ + + lr =__gic_v3_get_lr (i); +if (!(lr &ICH_LR_STATE )) + irq->active = false; + +vgic_put_irq (vcpu->kvm, irq); + } +} +void vgic_v3_load_nested (struct kvm_vcpu *vcpu) {struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
arch/arm64/kvm/vgic/vgic.c +4 -2
@@ -876,9 +876,11 @@ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) {int used_lrs; -/* If nesting, this is a load/put affair, not flush/sync. */ -if (vgic_state_is_nested (vcpu)) +/* If nesting, emulate the HW effect from L0 to L1 */ +if (vgic_state_is_nested (vcpu)) { +vgic_v3_sync_nested (vcpu);return ; + }/* An empty ap_list_head implies used_lrs == 0 */ if (list_empty (&vcpu->arch.vgic_cpu.ap_list_head))
include/kvm/arm_vgic.h +1 -0
@@ -391,6 +391,7 @@ void kvm_vgic_load(struct kvm_vcpu *vcpu);void kvm_vgic_put (struct kvm_vcpu *vcpu);void kvm_vgic_vmcr_sync (struct kvm_vcpu *vcpu); +void vgic_v3_sync_nested (struct kvm_vcpu *vcpu);void vgic_v3_load_nested (struct kvm_vcpu *vcpu);void vgic_v3_put_nested (struct kvm_vcpu *vcpu);void vgic_v3_handle_nested_maint_irq (struct kvm_vcpu *vcpu);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 54/69 KVM: arm64: nv: vgic: Allow userland to set VGIC maintenance IRQ
In reply to Marc ZyngierFrom: Andre Przywara <[email protected]>
The VGIC maintenance IRQ signals various conditions about the LRs, when the GIC's virtualization extension is used.
So far we didn't need it, but nested virtualization needs to know about this interrupt, so add a userland interface to setup the IRQ number. The architecture mandates that it must be a PPI, on top of that this code only exports a per-device option, so the PPI is the same on all VCPUs.
--- .../virt/kvm/devices/arm-vgic-v3.rst | 12 +++++++++- arch/arm64/include/uapi/asm/kvm.h | 1 + arch/arm64/kvm/vgic/vgic-kvm-device.c | 22 +++++++++++++++++++ include/kvm/arm_vgic.h | 3 +++ tools/arch/arm/include/uapi/asm/kvm.h | 1 + 5 files changed, 38 insertions(+), 1 deletion(-)
Documentation/virt/kvm/devices/arm-vgic-v3.rst +11 -1
@@ -284,8 +284,18 @@ Groups: | Aff3 | Aff2 | Aff1 | Aff0 | Errors: - ======= ============================================= -EINVAL vINTID is not multiple of 32 or info field is not VGIC_LEVEL_INFO_LINE_LEVEL ======= ============================================= + + KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ + Attributes: + + The attr field of kvm_device_attr encodes the following values: + + bits: | 31 .... 5 | 4 .... 0 | + values: | RES0 | vINTID | + + The vINTID specifies which interrupt is generated when the vGIC + must generate a maintenance interrupt. This must be a PPI.
arch/arm64/include/uapi/asm/kvm.h +1 -0
@@ -346,6 +346,7 @@ struct kvm_arm_copy_mte_tags {#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 +#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
arch/arm64/kvm/vgic/vgic-kvm-device.c +22 -0
@@ -251,6 +251,12 @@ static int vgic_get_common_attr(struct kvm_device *dev, VGIC_NR_PRIVATE_IRQS, uaddr);break ; } +case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ : { +u32 __user *uaddr = (u32 __user *)(long )attr->addr; + + r =put_user (dev->kvm->arch.vgic.maint_irq, uaddr); +break ; + } }return r;
@@ -637,6 +643,21 @@ static int vgic_v3_set_attr(struct kvm_device *dev, reg = tmp32;return vgic_v3_attr_regs_access (dev, attr, ®, true); } +case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ : { +u32 __user *uaddr = (u32 __user *)(long )attr->addr; +u32 val; + +if (get_user (val, uaddr)) +return -EFAULT ; + +/* Must be a PPI. */ +if ((val >=VGIC_NR_PRIVATE_IRQS ) || (val <VGIC_NR_SGIS )) +return -EINVAL ; + + dev->kvm->arch.vgic.maint_irq = val; + +return 0 ; + }case KVM_DEV_ARM_VGIC_GRP_CTRL : {int ret;
@@ -722,6 +743,7 @@ static int vgic_v3_has_attr(struct kvm_device *dev,case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS :return vgic_v3_has_attr_regs (dev, attr);case KVM_DEV_ARM_VGIC_GRP_NR_IRQS : +case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ :return 0 ;case KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO : { if (((attr->attr & KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK) >>
include/kvm/arm_vgic.h +3 -0
@@ -238,6 +238,9 @@ struct vgic_dist {int nr_spis; +/* The GIC maintenance IRQ for nested hypervisors. */ +u32 maint_irq; +/* base addresses in guest physical address space: */ gpa_t vgic_dist_base;/* distributor */ union {
tools/arch/arm/include/uapi/asm/kvm.h +1 -0
@@ -246,6 +246,7 @@ struct kvm_vcpu_events {#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 +#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 55/69 KVM: arm64: nv: Implement maintenance interrupt forwarding
In reply to Marc ZyngierWhen we take a maintenance interrupt, we need to decide whether it is generated on an action from the guest, or if it is something that needs to be forwarded to the guest hypervisor.
--- arch/arm64/kvm/vgic/vgic-init.c | 30 ++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v3-nested.c | 25 +++++++++++++++++++---- 2 files changed, 51 insertions(+), 4 deletions(-)
arch/arm64/kvm/vgic/vgic-init.c +30 -0
@@ -6,10 +6,12 @@#include <linux/uaccess.h> #include <linux/interrupt.h> #include <linux/cpu.h> +#include <linux/irq.h> #include <linux/kvm_host.h> #include <kvm/arm_vgic.h> #include <asm/kvm_emulate.h> #include <asm/kvm_mmu.h> +#include <asm/kvm_nested.h> #include "vgic.h" /*
@@ -221,6 +223,16 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)if (!irqchip_in_kernel (vcpu->kvm))return 0 ; +if (nested_virt_in_use (vcpu)) { +/* FIXME: remove this hack */ +if (vcpu->kvm->arch.vgic.maint_irq ==0 ) + vcpu->kvm->arch.vgic.maint_irq = kvm_vgic_global_state.maint_irq; + ret =kvm_vgic_set_owner (vcpu, vcpu->kvm->arch.vgic.maint_irq, + vcpu); +if (ret) +return ret; + } +/* * If we are creating a VCPU with a GICv3 we must also register the * KVM io device for the redistributor that belongs to this VCPU.
@@ -473,12 +485,23 @@ static int vgic_init_cpu_dying(unsigned int cpu)static irqreturn_t vgic_maintenance_handler (int irq,void *data) { +struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)data; +/* * We cannot rely on the vgic maintenance interrupt to be * delivered synchronously. This means we can only use it to * exit the VM, and we perform the handling of EOIed * interrupts on the exit path (see vgic_fold_lr_state). */ + +/* If not nested, deactivate */ +if (!vcpu || !vgic_state_is_nested (vcpu)) { +irq_set_irqchip_state (irq,IRQCHIP_STATE_ACTIVE , false); +return IRQ_HANDLED ; + } + +/* Assume nested from now */ +vgic_v3_handle_nested_maint_irq (vcpu);return IRQ_HANDLED ; }
@@ -577,6 +600,13 @@ int kvm_vgic_hyp_init(void)return ret; } + ret =irq_set_vcpu_affinity (kvm_vgic_global_state.maint_irq, +kvm_get_running_vcpus ()); +if (ret) { +kvm_err ("Error setting vcpu affinity\n" ); + goto out_free_irq; + } + ret =cpuhp_setup_state (CPUHP_AP_KVM_ARM_VGIC_INIT_STARTING ,"kvm/arm/vgic:starting" , vgic_init_cpu_starting, vgic_init_cpu_dying);
arch/arm64/kvm/vgic/vgic-v3-nested.c +21 -4
@@ -173,10 +173,20 @@ void vgic_v3_sync_nested(struct kvm_vcpu *vcpu)void vgic_v3_load_nested (struct kvm_vcpu *vcpu) {struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; +struct vgic_irq *irq; +unsigned long flags; vgic_cpu->shadow_vgic_v3 = vgic_cpu->nested_vgic_v3;vgic_v3_create_shadow_lr (vcpu);__vgic_v3_restore_state (vcpu_shadow_if (vcpu)); + + irq =vgic_get_irq (vcpu->kvm, vcpu, vcpu->kvm->arch.vgic.maint_irq); +raw_spin_lock_irqsave (&irq->irq_lock, flags); +if (irq->line_level || irq->active) +irq_set_irqchip_state (kvm_vgic_global_state.maint_irq, +IRQCHIP_STATE_ACTIVE , true); +raw_spin_unlock_irqrestore (&irq->irq_lock, flags); +vgic_put_irq (vcpu->kvm, irq); }void vgic_v3_put_nested (struct kvm_vcpu *vcpu)
@@ -191,11 +201,14 @@ void vgic_v3_put_nested(struct kvm_vcpu *vcpu)*/ vgic_v3_fixup_shadow_lr_state (vcpu); vgic_cpu->nested_vgic_v3 = vgic_cpu->shadow_vgic_v3; +irq_set_irqchip_state (kvm_vgic_global_state.maint_irq, +IRQCHIP_STATE_ACTIVE , false); }void vgic_v3_handle_nested_maint_irq (struct kvm_vcpu *vcpu) {struct vgic_v3_cpu_if *cpu_if =vcpu_nested_if (vcpu); +bool state;/* * If we exit a nested VM with a pending maintenance interrupt from the
@@ -203,8 +216,12 @@ void vgic_v3_handle_nested_maint_irq(struct kvm_vcpu *vcpu)* can re-sync the appropriate LRs and sample level triggered interrupts * again. */ -if (vgic_state_is_nested (vcpu) && - (cpu_if->vgic_hcr &ICH_HCR_EN ) && -vgic_v3_get_misr (vcpu)) -kvm_inject_nested_irq (vcpu); +if (!vgic_state_is_nested (vcpu)) +return ; + + state = cpu_if->vgic_hcr &ICH_HCR_EN ; + state &=vgic_v3_get_misr (vcpu); + +kvm_vgic_inject_irq (vcpu->kvm, vcpu->vcpu_id, + vcpu->kvm->arch.vgic.maint_irq, state, vcpu); }
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 56/69 KVM: arm64: nv: Add nested GICv3 tracepoints
In reply to Marc ZyngierFrom: Christoffer Dall <[email protected]>
Adding tracepoints to be able to peek into the shadow LRs used when running a guest guest.
--- arch/arm64/kvm/vgic/vgic-nested-trace.h | 137 ++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic-v3-nested.c | 13 ++- 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kvm/vgic/vgic-nested-trace.h
arch/arm64/kvm/vgic/vgic-nested-trace.h new file +137 -0
@@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(_TRACE_VGIC_NESTED_H) || defined(TRACE_HEADER_MULTI_READ ) +#define _TRACE_VGIC_NESTED_H + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +#define SLR_ENTRY_VALS (x) \ + " ", \ + !!(__entry->lrs[x] & ICH_LR_HW), \ + !!(__entry->lrs[x] & ICH_LR_PENDING_BIT), \ + !!(__entry->lrs[x] & ICH_LR_ACTIVE_BIT), \ + __entry->lrs[x] & ICH_LR_VIRTUAL_ID_MASK, \ + (__entry->lrs[x] & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT, \ + (__entry->orig_lrs[x] & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT + +TRACE_EVENT (vgic_create_shadow_lrs, + TP_PROTO(struct kvm_vcpu *vcpu, int nr_lr, u64 *lrs, u64 *orig_lrs), +TP_ARGS (vcpu, nr_lr, lrs, orig_lrs), + +TP_STRUCT__entry ( + __field( int, nr_lr ) + __array( u64, lrs, 16 ) +__array ( u64, orig_lrs,16 ) + ), + +TP_fast_assign ( + __entry->nr_lr = nr_lr; + memcpy(__entry->lrs, lrs, 16 * sizeof(u64)); + memcpy(__entry->orig_lrs, orig_lrs, 16 * sizeof(u64)); + ), + +TP_printk ("nr_lr: %d\n" +"%50sLR[ 0]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 1]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 2]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 3]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 4]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 5]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 6]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 7]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 8]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[ 9]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[10]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[11]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[12]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[13]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[14]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)\n" +"%50sLR[15]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu (%5llu)" , + __entry->nr_lr, +SLR_ENTRY_VALS (0 ),SLR_ENTRY_VALS (1 ),SLR_ENTRY_VALS (2 ), +SLR_ENTRY_VALS (3 ),SLR_ENTRY_VALS (4 ),SLR_ENTRY_VALS (5 ), +SLR_ENTRY_VALS (6 ),SLR_ENTRY_VALS (7 ),SLR_ENTRY_VALS (8 ), +SLR_ENTRY_VALS (9 ),SLR_ENTRY_VALS (10 ),SLR_ENTRY_VALS (11 ), +SLR_ENTRY_VALS (12 ),SLR_ENTRY_VALS (13 ),SLR_ENTRY_VALS (14 ), +SLR_ENTRY_VALS (15 )) +); + +#define LR_ENTRY_VALS (x) \ + " ", \ + !!(__entry->lrs[x] & ICH_LR_HW), \ + !!(__entry->lrs[x] & ICH_LR_PENDING_BIT), \ + !!(__entry->lrs[x] & ICH_LR_ACTIVE_BIT), \ + __entry->lrs[x] & ICH_LR_VIRTUAL_ID_MASK, \ + (__entry->lrs[x] & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT + +TRACE_EVENT (vgic_put_nested, + TP_PROTO(struct kvm_vcpu *vcpu, int nr_lr, u64 *lrs), +TP_ARGS (vcpu, nr_lr, lrs), + +TP_STRUCT__entry ( + __field( int, nr_lr ) +__array ( u64, lrs,16 ) + ), + +TP_fast_assign ( + __entry->nr_lr = nr_lr; + memcpy(__entry->lrs, lrs, 16 * sizeof(u64)); + ), + +TP_printk ("nr_lr: %d\n" +"%50sLR[ 0]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 1]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 2]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 3]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 4]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 5]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 6]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 7]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 8]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[ 9]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[10]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[11]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[12]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[13]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[14]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu\n" +"%50sLR[15]: HW: %d P: %d: A: %d vINTID: %5llu pINTID: %5llu" , + __entry->nr_lr, +LR_ENTRY_VALS (0 ),LR_ENTRY_VALS (1 ),LR_ENTRY_VALS (2 ), +LR_ENTRY_VALS (3 ),LR_ENTRY_VALS (4 ),LR_ENTRY_VALS (5 ), +LR_ENTRY_VALS (6 ),LR_ENTRY_VALS (7 ),LR_ENTRY_VALS (8 ), +LR_ENTRY_VALS (9 ),LR_ENTRY_VALS (10 ),LR_ENTRY_VALS (11 ), +LR_ENTRY_VALS (12 ),LR_ENTRY_VALS (13 ),LR_ENTRY_VALS (14 ), +LR_ENTRY_VALS (15 )) +); + +TRACE_EVENT (vgic_nested_hw_emulate, + TP_PROTO(int lr, u64 lr_val, u32 l1_intid), +TP_ARGS (lr, lr_val, l1_intid), + +TP_STRUCT__entry ( + __field( int, lr ) + __field( u64, lr_val ) +__field ( u32, l1_intid ) + ), + +TP_fast_assign ( + __entry->lr = lr; + __entry->lr_val = lr_val; + __entry->l1_intid = l1_intid; + ), + +TP_printk ("lr: %d LR %llx L1 INTID: %u\n" , + __entry->lr, __entry->lr_val, __entry->l1_intid) +); + +#endif /* _TRACE_VGIC_NESTED_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH vgic/ +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE vgic-nested-trace + +/* This part must be outside protection */ +#include <trace/define_trace.h>
arch/arm64/kvm/vgic/vgic-v3-nested.c +12 -1
@@ -13,6 +13,9 @@#include "vgic.h" +#define CREATE_TRACE_POINTS +#include "vgic-nested-trace.h" +static inline struct vgic_v3_cpu_if *vcpu_nested_if (struct kvm_vcpu *vcpu) {return &vcpu->arch.vgic_cpu.nested_vgic_v3;
@@ -119,6 +122,9 @@ static void vgic_v3_create_shadow_lr(struct kvm_vcpu *vcpu) used_lrs = i +1 ; } +trace_vgic_create_shadow_lrs (vcpu, kvm_vgic_global_state.nr_lr, + s_cpu_if->vgic_lr, cpu_if->vgic_lr); + s_cpu_if->used_lrs = used_lrs; }
@@ -163,8 +169,10 @@ void vgic_v3_sync_nested(struct kvm_vcpu *vcpu)continue ;/* oh well, the guest hyp is broken */ lr =__gic_v3_get_lr (i); -if (!(lr &ICH_LR_STATE )) +if (!(lr &ICH_LR_STATE )) { +trace_vgic_nested_hw_emulate (i, lr, l1_irq); irq->active = false; + }vgic_put_irq (vcpu->kvm, irq); }
@@ -195,6 +203,9 @@ void vgic_v3_put_nested(struct kvm_vcpu *vcpu)__vgic_v3_save_state (vcpu_shadow_if (vcpu)); +trace_vgic_put_nested (vcpu, kvm_vgic_global_state.nr_lr, +vcpu_shadow_if (vcpu)->vgic_lr); +/* * Translate the shadow state HW fields back to the virtual ones * before copying the shadow struct back to the nested one.
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 57/69 KVM: arm64: nv: Allow userspace to request KVM_ARM_VCPU_NESTED_VIRT
In reply to Marc ZyngierSince we're (almost) feature complete, let's allow userspace to request KVM_ARM_VCPU_NESTED_VIRT by bumping the KVM_VCPU_MAX_FEATURES up. We also now advertise the feature to userspace with a new capability.
It's going to be great...
--- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/kvm/arm.c | 3 +++ include/uapi/linux/kvm.h | 1 + 3 files changed, 5 insertions(+), 1 deletion(-)
arch/arm64/include/asm/kvm_host.h +1 -1
@@ -38,7 +38,7 @@#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS -#define KVM_VCPU_MAX_FEATURES 7 +#define KVM_VCPU_MAX_FEATURES 8#define KVM_REQ_SLEEP \ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
arch/arm64/kvm/arm.c +3 -0
@@ -257,6 +257,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)case KVM_CAP_ARM_EL1_32BIT : r =cpus_have_const_cap (ARM64_HAS_32BIT_EL1 );break ; +case KVM_CAP_ARM_EL2 : + r =cpus_have_const_cap (ARM64_HAS_NESTED_VIRT ); +break ;case KVM_CAP_GUEST_DEBUG_HW_BPS : r =get_num_brps ();break ;
include/uapi/linux/kvm.h +1 -0
@@ -1131,6 +1131,7 @@ struct kvm_ppc_resize_hpt {#define KVM_CAP_EXIT_ON_EMULATION_FAILURE 204#define KVM_CAP_ARM_MTE 205#define KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM 206 +#define KVM_CAP_ARM_EL2 207#ifdef KVM_CAP_IRQ_ROUTING
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 58/69 KVM: arm64: nv: Add handling of ARMv8.4-TTL TLB invalidation
In reply to Marc ZyngierSupport guest-provided information information to find out about the range of required invalidation.
--- arch/arm64/include/asm/kvm_nested.h | 1 + arch/arm64/kvm/nested.c | 57 +++++++++++++++++++++ arch/arm64/kvm/sys_regs.c | 78 ++++++++++++++++++----------- 3 files changed, 108 insertions(+), 28 deletions(-)
arch/arm64/include/asm/kvm_nested.h +1 -0
@@ -123,6 +123,7 @@ extern bool __forward_traps(struct kvm_vcpu *vcpu, unsigned int reg, u64 control_bit);extern bool forward_traps (struct kvm_vcpu *vcpu,u64 control_bit);extern bool forward_nv_traps (struct kvm_vcpu *vcpu); +unsigned int ttl_to_size (u8 ttl);struct sys_reg_params ;struct sys_reg_desc ;
arch/arm64/kvm/nested.c +57 -0
63 lines not on this page · raw message
arch/arm64/kvm/sys_regs.c +50 -28
109 lines not on this page · raw message
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v5 58/69] KVM: arm64: nv: Add handling of ARMv8.4-TTL TLB invalidation
Quoted 1 line · Marc Zyngier
Support guest-provided information information to find out about
Typo: information written twice.
Quoted 30 lines
the range of required invalidation.
Signed-off-by Marc Zyngier <[email protected]>--- arch/arm64/include/asm/kvm_nested.h | 1 + arch/arm64/kvm/nested.c | 57 +++++++++++++++++++++ arch/arm64/kvm/sys_regs.c | 78 ++++++++++++++++++----------- 3 files changed, 108 insertions(+), 28 deletions(-)arch/arm64/include/asm/kvm_nested.h +1
-0@@ -123,6 +123,7 @@ extern bool __forward_traps(struct kvm_vcpu *vcpu, unsigned int reg, u64 control_bit);extern bool forward_traps (struct kvm_vcpu *vcpu,u64 control_bit);extern bool forward_nv_traps (struct kvm_vcpu *vcpu); +unsigned int ttl_to_size (u8 ttl);struct sys_reg_params ;struct sys_reg_desc ;arch/arm64/kvm/nested.c +1
-0@@ -363,6 +363,63 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,return ret; } +
arch/arm64/kvm/nested.c int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, · line 376
Quoted 10 lines
arch/arm64/kvm/nested.c +10
-0+unsigned int ttl_to_size (u8 ttl) +{ +int level = ttl &3 ; +int gran = (ttl >>2 ) &3 ; +unsigned int max_size =0 ; + +switch (gran) { +case TLBI_TTL_TG_4K : +switch (level) { +case 0 :
/* No 52bit IPA support */
Quoted 6 lines
arch/arm64/kvm/nested.c +6
-0+break ; +case 1 : + max_size =SZ_1G ; +break ; +case 2 : + max_size =SZ_2M ;
arch/arm64/kvm/nested.c int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, · line 392
Quoted 10 lines
arch/arm64/kvm/nested.c +10
-0+break ; +case 3 : + max_size =SZ_4K ; +break ; + } +break ; +case TLBI_TTL_TG_16K : +switch (level) { +case 0 : +case 1 :
/* No 52bit IPA support */
Quoted 137 lines
arch/arm64/kvm/nested.c +30
-0+break ; +case 2 : + max_size =SZ_32M ; +break ; +case 3 : + max_size =SZ_16K ; +break ; + } +break ; +case TLBI_TTL_TG_64K : +switch (level) { +case 0 : +case 1 : +/* No 52bit IPA support */ +break ; +case 2 : + max_size =SZ_512M ; +break ; +case 3 : + max_size =SZ_64K ; +break ; + } +break ; +default :/* No size information */ +break ; + } + +return max_size; +} +/* Must be called with kvm->lock held */ struct kvm_s2_mmu *lookup_s2_mmu (struct kvm *kvm,u64 vttbr,u64 hcr) {arch/arm64/kvm/sys_regs.c +46
-27@@ -2699,59 +2699,81 @@ static bool handle_vmalls12e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p,return true; } +static unsigned long compute_tlb_inval_range (struct kvm_vcpu *vcpu, +struct kvm_s2_mmu *mmu, +u64 val) +{ +unsigned long max_size; +u8 ttl =0 ; + +if (cpus_have_const_cap (ARM64_HAS_ARMv8_4_TTL)) { + ttl =FIELD_GET (GENMASK_ULL (47 ,44 ), val); + } + + max_size =ttl_to_size (ttl); + +if (!max_size) { +u64 vtcr =vcpu_read_sys_reg (vcpu,VTCR_EL2 ); + +/* Compute the maximum extent of the invalidation */ +switch ((vtcr &VTCR_EL2_TG0_MASK )) { +case VTCR_EL2_TG0_4K : + max_size =SZ_1G ; +break ; +case VTCR_EL2_TG0_16K : + max_size =SZ_32M ; +break ; +case VTCR_EL2_TG0_64K : +/* +* No, we do not support 52bit IPA in nested yet. Once +* we do, this should be 4TB. +*/ +/* FIXME: remove the 52bit PA support from the IDregs */ + max_size =SZ_512M ; +break ; +default : +BUG (); + } + } + +WARN_ON (!max_size); +return max_size; +} +static bool handle_ipas2e1is (struct kvm_vcpu *vcpu,struct sys_reg_params *p,const struct sys_reg_desc *r) {u64 vttbr =vcpu_read_sys_reg (vcpu,VTTBR_EL2 ); -u64 vtcr =vcpu_read_sys_reg (vcpu,VTCR_EL2 );struct kvm_s2_mmu *mmu;u64 base_addr; -int max_size; +unsigned long max_size;/* * We drop a number of things from the supplied value: * * - NS bit: we're non-secure only. * -* - TTL field: We already have the granule size from the -* VTCR_EL2.TG0 field, and the level is only relevant to the -* guest's S2PT. -* * - IPA[51:48]: We don't support 52bit IPA just yet... * * And of course, adjust the IPA to be on an actual address. */ base_addr = (p->regval &GENMASK_ULL (35 ,0 )) <<12 ; -/* Compute the maximum extent of the invalidation */ -switch ((vtcr &VTCR_EL2_TG0_MASK )) { -case VTCR_EL2_TG0_4K : - max_size =SZ_1G ; -break ; -case VTCR_EL2_TG0_16K : - max_size =SZ_32M ; -break ; -case VTCR_EL2_TG0_64K : -/* -* No, we do not support 52bit IPA in nested yet. Once -* we do, this should be 4TB. -*/ -/* FIXME: remove the 52bit PA support from the IDregs */ - max_size =SZ_512M ; -break ; -default : -BUG (); - } -spin_lock (&vcpu->kvm->mmu_lock); mmu =lookup_s2_mmu (vcpu->kvm, vttbr,HCR_VM ); -if (mmu) +if (mmu) { + max_size =compute_tlb_inval_range (vcpu, mmu, p->regval); + base_addr &= ~(max_size -1 );kvm_unmap_stage2_range (mmu, base_addr, max_size); + }
arch/arm64/kvm/sys_regs.c static bool handle_vmalls12e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, · line 2779
Quoted 10 lines
arch/arm64/kvm/sys_regs.c +4
-1mmu =lookup_s2_mmu (vcpu->kvm, vttbr,0 ); -if (mmu) +if (mmu) { + max_size =compute_tlb_inval_range (vcpu, mmu, p->regval); + base_addr &= ~(max_size -1 );kvm_unmap_stage2_range (mmu, base_addr, max_size); + }spin_unlock (&vcpu->kvm->mmu_lock);
It looks good to me, please feel free to add.
Thanks,
Ganapat
Marc Zyngier
maintainerPATCH v5 59/69 KVM: arm64: nv: Invalidate TLBs based on shadow S2 TTL-like information
In reply to Marc ZyngierIn order to be able to make S2 TLB invalidations more performant on NV, let's use a scheme derived from the ARMv8.4 TTL extension.
If bits [56:55] in the descriptor are non-zero, they indicate a level which can be used as an invalidation range.
--- arch/arm64/include/asm/kvm_nested.h | 4 ++ arch/arm64/kvm/nested.c | 98 +++++++++++++++++++++++++++++ arch/arm64/kvm/sys_regs.c | 13 ++-- 3 files changed, 110 insertions(+), 5 deletions(-)
arch/arm64/include/asm/kvm_nested.h +4 -0
@@ -123,6 +123,8 @@ extern bool __forward_traps(struct kvm_vcpu *vcpu, unsigned int reg, u64 control_bit);extern bool forward_traps (struct kvm_vcpu *vcpu,u64 control_bit);extern bool forward_nv_traps (struct kvm_vcpu *vcpu); +u8 get_guest_mapping_ttl (struct kvm_vcpu *vcpu,struct kvm_s2_mmu *mmu, +u64 addr);unsigned int ttl_to_size (u8 ttl);struct sys_reg_params ;
@@ -131,4 +133,6 @@ struct sys_reg_desc;void access_nested_id_reg (struct kvm_vcpu *v,struct sys_reg_params *p,const struct sys_reg_desc *r); +#define KVM_NV_GUEST_MAP_SZ GENMASK_ULL(56, 55) +#endif /* __ARM64_KVM_NESTED_H */
arch/arm64/kvm/nested.c +98 -0
116 lines not on this page · raw message
arch/arm64/kvm/sys_regs.c +8 -5
@@ -2704,10 +2704,13 @@ static unsigned long compute_tlb_inval_range(struct kvm_vcpu *vcpu, u64 val) {unsigned long max_size; -u8 ttl =0 ; +u8 ttl; -if (cpus_have_const_cap (ARM64_HAS_ARMv8_4_TTL)) { - ttl =FIELD_GET (GENMASK_ULL (47 ,44 ), val); + ttl =FIELD_GET (GENMASK_ULL (47 ,44 ), val); + +if (!(cpus_have_const_cap (ARM64_HAS_ARMv8_4_TTL) && ttl)) { +u64 addr = (val &GENMASK_ULL (35 ,0 )) <<12 ; + ttl =get_guest_mapping_ttl (vcpu, mmu, addr); } max_size =ttl_to_size (ttl);
@@ -2748,6 +2751,8 @@ static bool handle_ipas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p,u64 base_addr;unsigned long max_size; +spin_lock (&vcpu->kvm->mmu_lock); +/* * We drop a number of things from the supplied value: *
@@ -2759,8 +2764,6 @@ static bool handle_ipas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p,*/ base_addr = (p->regval &GENMASK_ULL (35 ,0 )) <<12 ; -spin_lock (&vcpu->kvm->mmu_lock); - mmu =lookup_s2_mmu (vcpu->kvm, vttbr,HCR_VM );if (mmu) { max_size =compute_tlb_inval_range (vcpu, mmu, p->regval);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 60/69 KVM: arm64: nv: Tag shadow S2 entries with nested level
In reply to Marc ZyngierPopulate bits [56:55] of the leaf entry with the level provided by the guest's S2 translation.
--- arch/arm64/include/asm/kvm_nested.h | 8 ++++++++ arch/arm64/kvm/mmu.c | 6 ++++++ 2 files changed, 14 insertions(+)
arch/arm64/include/asm/kvm_nested.h +8 -0
@@ -5,6 +5,8 @@#include <linux/bitfield.h> #include <linux/kvm_host.h> +#include <asm/kvm_pgtable.h> +static inline bool nested_virt_in_use (const struct kvm_vcpu *vcpu) { return (!__is_defined(__KVM_NVHE_HYPERVISOR__) &&
@@ -135,4 +137,10 @@ void access_nested_id_reg(struct kvm_vcpu *v, struct sys_reg_params *p,#define KVM_NV_GUEST_MAP_SZ GENMASK_ULL(56, 55) +static inline u64 kvm_encode_nested_level (struct kvm_s2_trans *trans) +{ +return FIELD_PREP (KVM_PGTABLE_PROT_SW1 |KVM_PGTABLE_PROT_SW0 , + trans->level); +} +#endif /* __ARM64_KVM_NESTED_H */
arch/arm64/kvm/mmu.c +6 -0
@@ -1157,11 +1157,17 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,* Potentially reduce shadow S2 permissions to match the guest's own * S2. For exec faults, we'd only reach this point if the guest * actually allowed it (see kvm_s2_handle_perm_fault). +* +* Also encode the level of the nested translation in the SW bits of +* the PTE/PMD/PUD. This will be retrived on TLB invalidation from +* the guest. */ if (kvm_is_shadow_s2_fault (vcpu)) { writable &=kvm_s2_trans_writable (nested);if (!kvm_s2_trans_readable (nested)) prot &= ~KVM_PGTABLE_PROT_R ; + + prot |=kvm_encode_nested_level (nested); }spin_lock (&kvm->mmu_lock);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 61/69 KVM: arm64: nv: Add include containing the VNCR_EL2 offsets
In reply to Marc ZyngierVNCR_EL2 points to a page containing a number of system registers accessed by a guest hypervisor when ARMv8.4-NV is enabled.
Let's document the offsets in that page, as we are going to use this layout.
--- arch/arm64/include/asm/vncr_mapping.h | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 arch/arm64/include/asm/vncr_mapping.h
arch/arm64/include/asm/vncr_mapping.h new file +74 -0
@@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* System register offsets in the VNCR page +* All offsets are *byte* displacements! +*/ + +#ifndef __ARM64_VNCR_MAPPING_H__ +#define __ARM64_VNCR_MAPPING_H__ + +#define VNCR_VTTBR_EL2 0x020 +#define VNCR_VTCR_EL2 0x040 +#define VNCR_VMPIDR_EL2 0x050 +#define VNCR_CNTVOFF_EL2 0x060 +#define VNCR_HCR_EL2 0x078 +#define VNCR_HSTR_EL2 0x080 +#define VNCR_VPIDR_EL2 0x088 +#define VNCR_TPIDR_EL2 0x090 +#define VNCR_VNCR_EL2 0x0B0 +#define VNCR_CPACR_EL1 0x100 +#define VNCR_CONTEXTIDR_EL1 0x108 +#define VNCR_SCTLR_EL1 0x110 +#define VNCR_ACTLR_EL1 0x118 +#define VNCR_TCR_EL1 0x120 +#define VNCR_AFSR0_EL1 0x128 +#define VNCR_AFSR1_EL1 0x130 +#define VNCR_ESR_EL1 0x138 +#define VNCR_MAIR_EL1 0x140 +#define VNCR_AMAIR_EL1 0x148 +#define VNCR_MDSCR_EL1 0x158 +#define VNCR_SPSR_EL1 0x160 +#define VNCR_CNTV_CVAL_EL0 0x168 +#define VNCR_CNTV_CTL_EL0 0x170 +#define VNCR_CNTP_CVAL_EL0 0x178 +#define VNCR_CNTP_CTL_EL0 0x180 +#define VNCR_SCXTNUM_EL1 0x188 +#define VNCR_TFSR_EL1 0x190 +#define VNCR_ZCR_EL1 0x1E0 +#define VNCR_TTBR0_EL1 0x200 +#define VNCR_TTBR1_EL1 0x210 +#define VNCR_FAR_EL1 0x220 +#define VNCR_ELR_EL1 0x230 +#define VNCR_SP_EL1 0x240 +#define VNCR_VBAR_EL1 0x250 +#define VNCR_ICH_LR0_EL2 0x400 +// VNCR_ICH_LRN_EL2(n) VNCR_ICH_LR0_EL2+8*((n) & 7) +#define VNCR_ICH_AP0R0_EL2 0x480 +// VNCR_ICH_AP0RN_EL2(n) VNCR_ICH_AP0R0_EL2+8*((n) & 3) +#define VNCR_ICH_AP1R0_EL2 0x4A0 +// VNCR_ICH_AP1RN_EL2(n) VNCR_ICH_AP1R0_EL2+8*((n) & 3) +#define VNCR_ICH_HCR_EL2 0x4C0 +#define VNCR_ICH_VMCR_EL2 0x4C8 +#define VNCR_VDISR_EL2 0x500 +#define VNCR_PMBLIMITR_EL1 0x800 +#define VNCR_PMBPTR_EL1 0x810 +#define VNCR_PMBSR_EL1 0x820 +#define VNCR_PMSCR_EL1 0x828 +#define VNCR_PMSEVFR_EL1 0x830 +#define VNCR_PMSICR_EL1 0x838 +#define VNCR_PMSIRR_EL1 0x840 +#define VNCR_PMSLATFR_EL1 0x848 +#define VNCR_TRFCR_EL1 0x880 +#define VNCR_MPAM1_EL1 0x900 +#define VNCR_MPAMHCR_EL2 0x930 +#define VNCR_MPAMVPMV_EL2 0x938 +#define VNCR_MPAMVPM0_EL2 0x940 +#define VNCR_MPAMVPM1_EL2 0x948 +#define VNCR_MPAMVPM2_EL2 0x950 +#define VNCR_MPAMVPM3_EL2 0x958 +#define VNCR_MPAMVPM4_EL2 0x960 +#define VNCR_MPAMVPM5_EL2 0x968 +#define VNCR_MPAMVPM6_EL2 0x970 +#define VNCR_MPAMVPM7_EL2 0x978 + +#endif /* __ARM64_VNCR_MAPPING_H__ */
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 62/69 KVM: arm64: nv: Map VNCR-capable registers to a separate page
In reply to Marc ZyngierWith ARMv8.4-NV, registers that can be directly accessed in memory by the guest have to live at architected offsets in a special page.
Let's annotate the sysreg enum to reflect the offset at which they are in this page, whith a little twist:
If running on HW that doesn't have the ARMv8.4-NV feature, or even a VM that doesn't use NV, we store all the system registers in the usual sys_regs array. The only difference with the pre-8.4 situation is that VNCR-capable registers are at a "similar" offset as in the VNCR page (we can compute the actual offset at compile time), and that the sys_regs array is both bigger and sparse.
--- arch/arm64/include/asm/kvm_host.h | 99 ++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 35 deletions(-)
arch/arm64/include/asm/kvm_host.h +64 -35
@@ -27,6 +27,7 @@#include <asm/kvm.h> #include <asm/kvm_asm.h> #include <asm/thread_info.h> +#include <asm/vncr_mapping.h> #define __KVM_HAVE_ARCH_INTC_INITIALIZED
@@ -181,31 +182,32 @@ struct kvm_vcpu_fault_info {u64 disr_el1;/* Deferred [SError] Status Register */ }; +/* +* VNCR() just places the VNCR_capable registers in the enum after +* __VNCR_START__, and the value (after correction) to be an 8-byte offset +* from the VNCR base. As we don't require the enum to be otherwise ordered, +* we need the terrible hack below to ensure that we correctly size the +* sys_regs array, no matter what. +* +* The __MAX__ macro has been lifted from Sean Eron Anderson's wonderful +* treasure trove of bit hacks: +* https://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax +*/ +#define __MAX__ (x,y) ((x) ^ (((x) ^ (y)) & -((x) < (y)))) +#define VNCR (r) \ + __before_##r, \ + r = __VNCR_START__ + ((VNCR_ ## r) / 8), \ + __after_##r = __MAX__(__before_##r - 1, r) +enum vcpu_sysreg { __INVALID_SYSREG__,/* 0 is reserved as an invalid value */ MPIDR_EL1 ,/* MultiProcessor Affinity Register */ CSSELR_EL1 ,/* Cache Size Selection Register */ -SCTLR_EL1 ,/* System Control Register */ -ACTLR_EL1 ,/* Auxiliary Control Register */ -CPACR_EL1 ,/* Coprocessor Access Control */ -ZCR_EL1 ,/* SVE Control */ -TTBR0_EL1 ,/* Translation Table Base Register 0 */ -TTBR1_EL1 ,/* Translation Table Base Register 1 */ -TCR_EL1 ,/* Translation Control Register */ -ESR_EL1 ,/* Exception Syndrome Register */ -AFSR0_EL1 ,/* Auxiliary Fault Status Register 0 */ -AFSR1_EL1 ,/* Auxiliary Fault Status Register 1 */ -FAR_EL1 ,/* Fault Address Register */ -MAIR_EL1 ,/* Memory Attribute Indirection Register */ -VBAR_EL1 ,/* Vector Base Address Register */ -CONTEXTIDR_EL1 ,/* Context ID Register */ TPIDR_EL0 ,/* Thread ID, User R/W */ TPIDRRO_EL0 ,/* Thread ID, User R/O */ TPIDR_EL1 ,/* Thread ID, Privileged */ -AMAIR_EL1 ,/* Aux Memory Attribute Indirection Register */ CNTKCTL_EL1 ,/* Timer Control Register (EL1) */ PAR_EL1 ,/* Physical Address Register */ -MDSCR_EL1 ,/* Monitor Debug System Control Register */ MDCCINT_EL1 ,/* Monitor Debug Comms Channel Interrupt Enable Reg */ DISR_EL1 ,/* Deferred Interrupt Status Register */
@@ -235,20 +237,9 @@ enum vcpu_sysreg {APGAKEYLO_EL1 ,APGAKEYHI_EL1 , -ELR_EL1 , -SP_EL1 , -SPSR_EL1 , - -CNTVOFF_EL2 , -CNTV_CVAL_EL0 , -CNTV_CTL_EL0 , -CNTP_CVAL_EL0 , -CNTP_CTL_EL0 , -/* Memory Tagging Extension registers */ RGSR_EL1 ,/* Random Allocation Tag Seed Register */ GCR_EL1 ,/* Tag Control Register */ -TFSR_EL1 ,/* Tag Fault Status Register (EL1) */ TFSRE0_EL1, /* Tag Fault Status Register (EL0) *//* 32bit specific registers. */
@@ -258,20 +249,14 @@ enum vcpu_sysreg {DBGVCR32_EL2 ,/* Debug Vector Catch Register */ /* EL2 registers */ -VPIDR_EL2 ,/* Virtualization Processor ID Register */ -VMPIDR_EL2 ,/* Virtualization Multiprocessor ID Register */ SCTLR_EL2 ,/* System Control Register (EL2) */ ACTLR_EL2 ,/* Auxiliary Control Register (EL2) */ -HCR_EL2 ,/* Hypervisor Configuration Register */ MDCR_EL2 ,/* Monitor Debug Configuration Register (EL2) */ CPTR_EL2 ,/* Architectural Feature Trap Register (EL2) */ -HSTR_EL2 ,/* Hypervisor System Trap Register */ HACR_EL2 ,/* Hypervisor Auxiliary Control Register */ TTBR0_EL2 ,/* Translation Table Base Register 0 (EL2) */ TTBR1_EL2 ,/* Translation Table Base Register 1 (EL2) */ TCR_EL2 ,/* Translation Control Register (EL2) */ -VTTBR_EL2 ,/* Virtualization Translation Table Base Register */ -VTCR_EL2 ,/* Virtualization Translation Control Register */ SPSR_EL2 ,/* EL2 saved program status register */ ELR_EL2 ,/* EL2 exception link register */ AFSR0_EL2, /* Auxiliary Fault Status Register 0 (EL2) */
@@ -284,7 +269,6 @@ enum vcpu_sysreg {VBAR_EL2 ,/* Vector Base Address Register (EL2) */ RVBAR_EL2 ,/* Reset Vector Base Address Register */ CONTEXTIDR_EL2 ,/* Context ID Register (EL2) */ -TPIDR_EL2 ,/* EL2 Software Thread ID Register */ CNTHCTL_EL2 ,/* Counter-timer Hypervisor Control register */ SP_EL2 ,/* EL2 Stack Pointer */ CNTHP_CTL_EL2,
@@ -292,6 +276,42 @@ enum vcpu_sysreg {CNTHV_CTL_EL2 ,CNTHV_CVAL_EL2 , + __VNCR_START__,/* Any VNCR-capable reg goes after this point */ + +VNCR (SCTLR_EL1 ),/* System Control Register */ +VNCR (ACTLR_EL1 ),/* Auxiliary Control Register */ +VNCR (CPACR_EL1 ),/* Coprocessor Access Control */ +VNCR (ZCR_EL1 ),/* SVE Control */ +VNCR (TTBR0_EL1 ),/* Translation Table Base Register 0 */ +VNCR (TTBR1_EL1 ),/* Translation Table Base Register 1 */ +VNCR (TCR_EL1 ),/* Translation Control Register */ +VNCR (ESR_EL1 ),/* Exception Syndrome Register */ +VNCR (AFSR0_EL1 ),/* Auxiliary Fault Status Register 0 */ +VNCR (AFSR1_EL1 ),/* Auxiliary Fault Status Register 1 */ +VNCR (FAR_EL1 ),/* Fault Address Register */ +VNCR (MAIR_EL1 ),/* Memory Attribute Indirection Register */ +VNCR (VBAR_EL1 ),/* Vector Base Address Register */ +VNCR (CONTEXTIDR_EL1 ),/* Context ID Register */ +VNCR (AMAIR_EL1 ),/* Aux Memory Attribute Indirection Register */ +VNCR (MDSCR_EL1 ),/* Monitor Debug System Control Register */ +VNCR (ELR_EL1 ), +VNCR (SP_EL1 ), +VNCR (SPSR_EL1 ), +VNCR (TFSR_EL1 ),/* Tag Fault Status Register (EL1) */ +VNCR (VPIDR_EL2 ),/* Virtualization Processor ID Register */ +VNCR (VMPIDR_EL2 ),/* Virtualization Multiprocessor ID Register */ +VNCR (HCR_EL2 ),/* Hypervisor Configuration Register */ +VNCR (HSTR_EL2 ),/* Hypervisor System Trap Register */ +VNCR (VTTBR_EL2 ),/* Virtualization Translation Table Base Register */ +VNCR (VTCR_EL2 ),/* Virtualization Translation Control Register */ +VNCR (TPIDR_EL2 ),/* EL2 Software Thread ID Register */ + +VNCR (CNTVOFF_EL2 ), +VNCR (CNTV_CVAL_EL0 ), +VNCR (CNTV_CTL_EL0 ), +VNCR (CNTP_CVAL_EL0 ), +VNCR (CNTP_CTL_EL0 ), +NR_SYS_REGS /* Nothing after this line! */ };
@@ -308,6 +328,9 @@ struct kvm_cpu_context {u64 sys_regs[NR_SYS_REGS ];struct kvm_vcpu *__hyp_running_vcpu; + +/* This pointer has to be 4kB aligned. */ +u64 *vncr_array; };struct kvm_pmu_events {
@@ -541,7 +564,13 @@ struct kvm_vcpu_arch {* for system registers that are never context switched, but only * emulated. */ -#define __ctxt_sys_reg (c,r) (&(c)->sys_regs[(r)]) +static inline u64 *__ctxt_sys_reg (const struct kvm_cpu_context *ctxt,int r) +{ +if (unlikely (r >= __VNCR_START__ && ctxt->vncr_array)) +return &ctxt->vncr_array[r - __VNCR_START__]; + +return (u64 *)&ctxt->sys_regs[r]; +}#define ctxt_sys_reg (c,r) (*__ctxt_sys_reg(c,r))
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 63/69 KVM: arm64: nv: Move nested vgic state into the sysreg file
In reply to Marc ZyngierThe vgic nested state needs to be accessible from the VNCR page, and thus needs to be part of the normal sysreg file. Let's move it there.
--- arch/arm64/include/asm/kvm_host.h | 9 +++ arch/arm64/kvm/sys_regs.c | 53 +++++++++++------ arch/arm64/kvm/vgic/vgic-v3-nested.c | 88 ++++++++++++++-------------- arch/arm64/kvm/vgic/vgic-v3.c | 17 ++++-- arch/arm64/kvm/vgic/vgic.h | 10 ++++ include/kvm/arm_vgic.h | 7 --- 6 files changed, 110 insertions(+), 74 deletions(-)
arch/arm64/include/asm/kvm_host.h +9 -0
@@ -312,6 +312,15 @@ enum vcpu_sysreg {VNCR (CNTP_CVAL_EL0 ),VNCR (CNTP_CTL_EL0 ), +VNCR (ICH_LR0_EL2 ), +ICH_LR15_EL2 =ICH_LR0_EL2 +15 , +VNCR (ICH_AP0R0_EL2 ), +ICH_AP0R3_EL2 =ICH_AP0R0_EL2 +3 , +VNCR (ICH_AP1R0_EL2 ), +ICH_AP1R3_EL2 =ICH_AP1R0_EL2 +3 , +VNCR (ICH_HCR_EL2 ), +VNCR (ICH_VMCR_EL2 ), +NR_SYS_REGS /* Nothing after this line! */ };
arch/arm64/kvm/sys_regs.c +35 -18
93 lines not on this page · raw message
arch/arm64/kvm/vgic/vgic-v3-nested.c +45 -43
190 lines not on this page · raw message
arch/arm64/kvm/vgic/vgic-v3.c +11 -6
@@ -280,10 +280,11 @@ void vgic_v3_enable(struct kvm_vcpu *vcpu) ICC_SRE_EL1_SRE);/* * If nesting is allowed, force GICv3 onto the nested -* guests as well. +* guests as well by setting the shadow state to the +* same value. */ if (nested_virt_in_use (vcpu)) - vcpu->arch.vgic_cpu.nested_vgic_v3.vgic_sre = vgic_v3->vgic_sre; + vcpu->arch.vgic_cpu.shadow_vgic_v3.vgic_sre = vgic_v3->vgic_sre; vcpu->arch.vgic_cpu.pendbaser =INITIAL_PENDBASER_VALUE ; }else { vgic_v3->vgic_sre =0 ;
@@ -715,11 +716,15 @@ void vgic_v3_load(struct kvm_vcpu *vcpu)struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;/* -* vgic_v3_load_nested only affects the LRs in the shadow -* state, so it is fine to pass the nested state around. +* If the vgic is in nested state, populate the shadow state +* from the guest's nested state. As vgic_v3_load_nested() +* will only load LRs, let's deal with the rest of the state +* here as if it was a non-nested state. Cunning. */ -if (vgic_state_is_nested (vcpu)) - cpu_if = &vcpu->arch.vgic_cpu.nested_vgic_v3; +if (vgic_state_is_nested (vcpu)) { +vgic_v3_create_shadow_state (vcpu); + cpu_if = &vcpu->arch.vgic_cpu.shadow_vgic_v3; + }/* * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
arch/arm64/kvm/vgic/vgic.h +10 -0
@@ -323,4 +323,14 @@ void vgic_v4_teardown(struct kvm *kvm);void vgic_v4_configure_vsgis (struct kvm *kvm);void vgic_v4_get_vlpi_state (struct vgic_irq *irq,bool *val); +void vgic_v3_sync_nested (struct kvm_vcpu *vcpu); +void vgic_v3_create_shadow_state (struct kvm_vcpu *vcpu); +void vgic_v3_load_nested (struct kvm_vcpu *vcpu); +void vgic_v3_put_nested (struct kvm_vcpu *vcpu); +void vgic_v3_handle_nested_maint_irq (struct kvm_vcpu *vcpu); + +#define ICH_LRN (n) (ICH_LR0_EL2 + (n)) +#define ICH_AP0RN (n) (ICH_AP0R0_EL2 + (n)) +#define ICH_AP1RN (n) (ICH_AP1R0_EL2 + (n)) +#endif
include/kvm/arm_vgic.h +0 -7
@@ -328,9 +328,6 @@ struct vgic_cpu {struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS ]; -/* CPU vif control registers for the virtual GICH interface */ -struct vgic_v3_cpu_if nested_vgic_v3; -/* * The shadow vif control register loaded to the hardware when * running a nested L2 guest with the virtual IMO/FMO bit set.
@@ -394,10 +391,6 @@ void kvm_vgic_load(struct kvm_vcpu *vcpu);void kvm_vgic_put (struct kvm_vcpu *vcpu);void kvm_vgic_vmcr_sync (struct kvm_vcpu *vcpu); -void vgic_v3_sync_nested (struct kvm_vcpu *vcpu); -void vgic_v3_load_nested (struct kvm_vcpu *vcpu); -void vgic_v3_put_nested (struct kvm_vcpu *vcpu); -void vgic_v3_handle_nested_maint_irq (struct kvm_vcpu *vcpu);u16 vgic_v3_get_eisr (struct kvm_vcpu *vcpu);u16 vgic_v3_get_elrsr (struct kvm_vcpu *vcpu);u64 vgic_v3_get_misr (struct kvm_vcpu *vcpu);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 64/69 KVM: arm64: Add ARMv8.4 Enhanced Nested Virt cpufeature
In reply to Marc ZyngierAdd the detection code for the ARMv8.4-NV feature.
--- arch/arm64/include/asm/kvm_nested.h | 6 ++++++ arch/arm64/kernel/cpufeature.c | 10 ++++++++++ arch/arm64/tools/cpucaps | 1 + 3 files changed, 17 insertions(+)
arch/arm64/include/asm/kvm_nested.h +6 -0
@@ -14,6 +14,12 @@ static inline bool nested_virt_in_use(const struct kvm_vcpu *vcpu) test_bit(KVM_ARM_VCPU_HAS_EL2, vcpu->arch.features)); } +static inline bool enhanced_nested_virt_in_use (const struct kvm_vcpu *vcpu) +{ +return cpus_have_final_cap (ARM64_HAS_ENHANCED_NESTED_VIRT ) && +nested_virt_in_use (vcpu); +} +/* Translation helpers from non-VHE EL2 to EL1 */ static inline u64 tcr_el2_ips_to_tcr_el1_ps (u64 tcr_el2) {
arch/arm64/kernel/cpufeature.c +10 -0
@@ -2018,6 +2018,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .field_pos = ID_AA64MMFR2_NV_SHIFT, .min_field_value = 1, }, + { + .desc = "Enhanced Nested Virtualization Support", + .capability = ARM64_HAS_ENHANCED_NESTED_VIRT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_nested_virt_support, + .sys_reg = SYS_ID_AA64MMFR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64MMFR2_NV_SHIFT, + .min_field_value = 2, + }, { .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, .type = ARM64_CPUCAP_SYSTEM_FEATURE,
arch/arm64/tools/cpucaps +1 -0
@@ -19,6 +19,7 @@ HAS_DCPODP HAS_DCPOP HAS_E0PD HAS_ECV +HAS_ENHANCED_NESTED_VIRT HAS_EPAN HAS_GENERIC_AUTH HAS_GENERIC_AUTH_ARCH
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 65/69 KVM: arm64: nv: Sync nested timer state with ARMv8.4
In reply to Marc ZyngierFrom: Christoffer Dall <[email protected]>
Emulating the ARMv8.4-NV timers is a bit odd, as the timers can be reconfigured behind our back without the hypervisor even noticing. In the VHE case, that's an actual regression in the architecture...
--- arch/arm64/kvm/arch_timer.c | 37 ++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/arm.c | 3 +++ include/kvm/arm_arch_timer.h | 1 + 3 files changed, 41 insertions(+)
arch/arm64/kvm/arch_timer.c +37 -0
@@ -786,6 +786,43 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)set_cntvoff (0 ); } +void kvm_timer_sync_nested (struct kvm_vcpu *vcpu) +{ +if (!is_hyp_ctxt (vcpu)) +return ; + +/* +* Guest hypervisors using ARMv8.4 enhanced nested virt support have +* their EL1 timer register accesses redirected to the VNCR page. +*/ +if (!vcpu_el2_e2h_is_set (vcpu)) { +/* +* For a non-VHE guest hypervisor, we update the hardware +* timer registers with the latest value written by the guest +* to the VNCR page and let the hardware take care of the +* rest. +*/ +write_sysreg_el0 (__vcpu_sys_reg (vcpu,CNTV_CTL_EL0 ),SYS_CNTV_CTL ); +write_sysreg_el0 (__vcpu_sys_reg (vcpu,CNTV_CVAL_EL0 ),SYS_CNTV_CVAL ); +write_sysreg_el0 (__vcpu_sys_reg (vcpu,CNTP_CTL_EL0 ),SYS_CNTP_CTL ); +write_sysreg_el0 (__vcpu_sys_reg (vcpu,CNTP_CVAL_EL0 ),SYS_CNTP_CVAL ); + }else { +/* +* For a VHE guest hypervisor, the emulated state (which +* is stored in the VNCR page) could have been updated behind +* our back, and we must reset the emulation of the timers. +*/ + +struct timer_map map; +get_timer_map (vcpu, &map); + +soft_timer_cancel (&map.emul_vtimer->hrtimer); +soft_timer_cancel (&map.emul_ptimer->hrtimer); +timer_emulate (map.emul_vtimer); +timer_emulate (map.emul_ptimer); + } +} +/* * With a userspace irqchip we have to check if the guest de-asserted the * timer and if so, unmask the timer irq signal on the host interrupt
arch/arm64/kvm/arm.c +3 -0
@@ -902,6 +902,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)if (static_branch_unlikely (&userspace_irqchip_in_use))kvm_timer_sync_user (vcpu); +if (enhanced_nested_virt_in_use (vcpu)) +kvm_timer_sync_nested (vcpu); +kvm_arch_vcpu_ctxsync_fp (vcpu);/*
include/kvm/arm_arch_timer.h +1 -0
@@ -68,6 +68,7 @@ int kvm_timer_hyp_init(bool);int kvm_timer_enable (struct kvm_vcpu *vcpu);int kvm_timer_vcpu_reset (struct kvm_vcpu *vcpu);void kvm_timer_vcpu_init (struct kvm_vcpu *vcpu); +void kvm_timer_sync_nested (struct kvm_vcpu *vcpu);void kvm_timer_sync_user (struct kvm_vcpu *vcpu);bool kvm_timer_should_notify_user (struct kvm_vcpu *vcpu);void kvm_timer_update_run (struct kvm_vcpu *vcpu);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 66/69 KVM: arm64: nv: Allocate VNCR page when required
In reply to Marc ZyngierIf running a NV guest on an ARMv8.4-NV capable system, let's allocate an additional page that will be used by the hypervisor to fulfill system register accesses.
--- arch/arm64/include/asm/kvm_host.h | 3 ++- arch/arm64/kvm/nested.c | 10 ++++++++++ arch/arm64/kvm/reset.c | 1 + 3 files changed, 13 insertions(+), 1 deletion(-)
arch/arm64/include/asm/kvm_host.h +2 -1
@@ -575,7 +575,8 @@ struct kvm_vcpu_arch {*/ static inline u64 *__ctxt_sys_reg (const struct kvm_cpu_context *ctxt,int r) { -if (unlikely (r >= __VNCR_START__ && ctxt->vncr_array)) +if (unlikely (cpus_have_final_cap (ARM64_HAS_ENHANCED_NESTED_VIRT ) && + r >= __VNCR_START__ && ctxt->vncr_array))return &ctxt->vncr_array[r - __VNCR_START__];return (u64 *)&ctxt->sys_regs[r];
arch/arm64/kvm/nested.c +10 -0
@@ -47,6 +47,14 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)if (!cpus_have_final_cap (ARM64_HAS_NESTED_VIRT ))return -EINVAL ; +if (cpus_have_final_cap (ARM64_HAS_ENHANCED_NESTED_VIRT )) { +if (!vcpu->arch.ctxt.vncr_array) + vcpu->arch.ctxt.vncr_array = (u64 *)__get_free_page (GFP_KERNEL | __GFP_ZERO); + +if (!vcpu->arch.ctxt.vncr_array) +return -ENOMEM ; + } +mutex_lock (&kvm->lock);/*
@@ -76,6 +84,8 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) kvm_init_stage2_mmu(kvm, &tmp[num_mmus - 2])) { kvm_free_stage2_pgd(&tmp[num_mmus - 1]); kvm_free_stage2_pgd(&tmp[num_mmus - 2]); + free_page((unsigned long)vcpu->arch.ctxt.vncr_array); + vcpu->arch.ctxt.vncr_array = NULL; } else { kvm->arch.nested_mmus_size = num_mmus; ret =0 ;
arch/arm64/kvm/reset.c +1 -0
@@ -146,6 +146,7 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu)void kvm_arm_vcpu_destroy (struct kvm_vcpu *vcpu) {kfree (vcpu->arch.sve_state); +free_page ((unsigned long )vcpu->arch.ctxt.vncr_array); }static void kvm_vcpu_reset_sve (struct kvm_vcpu *vcpu)
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 67/69 KVM: arm64: nv: Enable ARMv8.4-NV support
In reply to Marc ZyngierAs all the VNCR-capable system registers are nicely separated from the rest of the crowd, let's set HCR_EL2.NV2 on and let the ball rolling.
--- arch/arm64/include/asm/kvm_arm.h | 1 + arch/arm64/include/asm/kvm_emulate.h | 23 +++++++++++++---------- arch/arm64/include/asm/sysreg.h | 1 + arch/arm64/kvm/hyp/vhe/switch.c | 14 +++++++++++++- 4 files changed, 28 insertions(+), 11 deletions(-)
arch/arm64/include/asm/kvm_arm.h +1 -0
@@ -20,6 +20,7 @@#define HCR_AMVOFFEN (UL(1) << 51)#define HCR_FIEN (UL(1) << 47)#define HCR_FWB (UL(1) << 46) +#define HCR_NV2 (UL(1) << 45)#define HCR_AT (UL(1) << 44)#define HCR_NV1 (UL(1) << 43)#define HCR_NV (UL(1) << 42)
arch/arm64/include/asm/kvm_emulate.h +13 -10
@@ -245,21 +245,24 @@ static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)static inline u64 __fixup_spsr_el2_write (struct kvm_cpu_context *ctxt,u64 val) { -if (!__vcpu_el2_e2h_is_set (ctxt)) { -/* -* Clear the .M field when writing SPSR to the CPU, so that we -* can detect when the CPU clobbered our SPSR copy during a -* local exception. -*/ - val &= ~0xc ; - } + struct kvm_vcpu *vcpu = container_of(ctxt, struct kvm_vcpu, arch.ctxt); + +if (enhanced_nested_virt_in_use (vcpu) ||__vcpu_el2_e2h_is_set (ctxt)) +return val; -return val; +/* +* Clear the .M field when writing SPSR to the CPU, so that we +* can detect when the CPU clobbered our SPSR copy during a +* local exception. +*/ +return val &= ~0xc ; }static inline u64 __fixup_spsr_el2_read (const struct kvm_cpu_context *ctxt,u64 val) { -if (__vcpu_el2_e2h_is_set (ctxt)) + struct kvm_vcpu *vcpu = container_of(ctxt, struct kvm_vcpu, arch.ctxt); + +if (enhanced_nested_virt_in_use (vcpu) ||__vcpu_el2_e2h_is_set (ctxt))return val;/*
arch/arm64/include/asm/sysreg.h +1 -0
@@ -550,6 +550,7 @@#define SYS_TCR_EL2 sys_reg(3, 4, 2, 0, 2)#define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)#define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) +#define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0)#define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0)#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1)
arch/arm64/kvm/hyp/vhe/switch.c +13 -1
@@ -45,7 +45,13 @@ static void __activate_traps(struct kvm_vcpu *vcpu)* the EL1 virtual memory control register accesses * as well as the AT S1 operations. */ - hcr |=HCR_TVM |HCR_TRVM |HCR_AT |HCR_TTLB |HCR_NV1 ; +if (enhanced_nested_virt_in_use (vcpu)) { + hcr &= ~HCR_TVM ; + }else { + hcr |=HCR_TVM |HCR_TRVM |HCR_TTLB ; + } + + hcr |=HCR_AT |HCR_NV1 ; }else {/* * For a guest hypervisor on v8.1 (VHE), allow to
@@ -79,6 +85,12 @@ static void __activate_traps(struct kvm_vcpu *vcpu)if (!vcpu_el2_tge_is_set (vcpu)) hcr |=HCR_AT |HCR_TTLB ; } + +if (enhanced_nested_virt_in_use (vcpu)) { + hcr |=HCR_AT |HCR_TTLB |HCR_NV2 ; +write_sysreg_s (vcpu->arch.ctxt.vncr_array, +SYS_VNCR_EL2 ); + } }else if (nested_virt_in_use (vcpu)) {u64 vhcr_el2 =__vcpu_sys_reg (vcpu,HCR_EL2 );
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v5 67/69] KVM: arm64: nv: Enable ARMv8.4-NV support
Quoted 80 lines · Marc Zyngier
As all the VNCR-capable system registers are nicely separated from the rest of the crowd, let's set HCR_EL2.NV2 on and let the ball rolling.
Signed-off-by Marc Zyngier <[email protected]>--- arch/arm64/include/asm/kvm_arm.h | 1 + arch/arm64/include/asm/kvm_emulate.h | 23 +++++++++++++---------- arch/arm64/include/asm/sysreg.h | 1 + arch/arm64/kvm/hyp/vhe/switch.c | 14 +++++++++++++- 4 files changed, 28 insertions(+), 11 deletions(-)arch/arm64/include/asm/kvm_arm.h +1
-0@@ -20,6 +20,7 @@#define HCR_AMVOFFEN (UL(1) << 51)#define HCR_FIEN (UL(1) << 47)#define HCR_FWB (UL(1) << 46) +#define HCR_NV2 (UL(1) << 45)#define HCR_AT (UL(1) << 44)#define HCR_NV1 (UL(1) << 43)#define HCR_NV (UL(1) << 42)arch/arm64/include/asm/kvm_emulate.h +13
-10@@ -245,21 +245,24 @@ static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)static inline u64 __fixup_spsr_el2_write (struct kvm_cpu_context *ctxt,u64 val) { -if (!__vcpu_el2_e2h_is_set (ctxt)) { -/* -* Clear the .M field when writing SPSR to the CPU, so that we -* can detect when the CPU clobbered our SPSR copy during a -* local exception. -*/ - val &= ~0xc ; - } + struct kvm_vcpu *vcpu = container_of(ctxt, struct kvm_vcpu, arch.ctxt); + +if (enhanced_nested_virt_in_use (vcpu) ||__vcpu_el2_e2h_is_set (ctxt)) +return val; -return val; +/* +* Clear the .M field when writing SPSR to the CPU, so that we +* can detect when the CPU clobbered our SPSR copy during a +* local exception. +*/ +return val &= ~0xc ; }static inline u64 __fixup_spsr_el2_read (const struct kvm_cpu_context *ctxt,u64 val) { -if (__vcpu_el2_e2h_is_set (ctxt)) + struct kvm_vcpu *vcpu = container_of(ctxt, struct kvm_vcpu, arch.ctxt); + +if (enhanced_nested_virt_in_use (vcpu) ||__vcpu_el2_e2h_is_set (ctxt))return val;/* arch/arm64/include/asm/sysreg.h +1
-0@@ -550,6 +550,7 @@#define SYS_TCR_EL2 sys_reg(3, 4, 2, 0, 2)#define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)#define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) +#define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0)#define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0)#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1)arch/arm64/kvm/hyp/vhe/switch.c +0
-0
arch/arm64/kvm/hyp/vhe/switch.c static void __activate_traps(struct kvm_vcpu *vcpu) · line 49
Quoted 6 lines
arch/arm64/kvm/hyp/vhe/switch.c +2
-1@@ -45,7 +45,13 @@ static void __activate_traps(struct kvm_vcpu *vcpu)* the EL1 virtual memory control register accesses * as well as the AT S1 operations. */ - hcr |=HCR_TVM |HCR_TRVM |HCR_AT |HCR_TTLB |HCR_NV1 ; +if (enhanced_nested_virt_in_use (vcpu)) { + hcr &= ~HCR_TVM ;
I think, we should clear TRVM also?
hcr &= ~(HCR_TVM | HCR_TRVM);
Quoted 11 lines
arch/arm64/kvm/hyp/vhe/switch.c +5
-0+ }else { + hcr |=HCR_TVM |HCR_TRVM |HCR_TTLB ; + } + + hcr |=HCR_AT |HCR_NV1 ; }else {/* * For a guest hypervisor on v8.1 (VHE), allow to @@ -79,6 +85,12 @@ static void __activate_traps(struct kvm_vcpu *vcpu)if (!vcpu_el2_tge_is_set (vcpu)) hcr |=HCR_AT |HCR_TTLB ;
arch/arm64/kvm/hyp/vhe/switch.c static void __activate_traps(struct kvm_vcpu *vcpu) · line 96
Quoted 10 lines
arch/arm64/kvm/hyp/vhe/switch.c +6
-0} + +if (enhanced_nested_virt_in_use (vcpu)) { + hcr |=HCR_AT |HCR_TTLB |HCR_NV2 ; +write_sysreg_s (vcpu->arch.ctxt.vncr_array, +SYS_VNCR_EL2 ); + } }else if (nested_virt_in_use (vcpu)) {u64 vhcr_el2 =__vcpu_sys_reg (vcpu,HCR_EL2 );
Thanks,
Ganapat
Marc Zyngier
maintainerRe: [PATCH v5 67/69] KVM: arm64: nv: Enable ARMv8.4-NV support
Quoted 89 lines · Ganapatrao Kulkarni
Quoted 77 lines · Marc Zyngier
As all the VNCR-capable system registers are nicely separated from the rest of the crowd, let's set HCR_EL2.NV2 on and let the ball rolling.
Signed-off-by Marc Zyngier <[email protected]>--- arch/arm64/include/asm/kvm_arm.h | 1 + arch/arm64/include/asm/kvm_emulate.h | 23 +++++++++++++---------- arch/arm64/include/asm/sysreg.h | 1 + arch/arm64/kvm/hyp/vhe/switch.c | 14 +++++++++++++- 4 files changed, 28 insertions(+), 11 deletions(-)arch/arm64/include/asm/kvm_arm.h +1
-0@@ -20,6 +20,7 @@#define HCR_AMVOFFEN (UL(1) << 51)#define HCR_FIEN (UL(1) << 47)#define HCR_FWB (UL(1) << 46) +#define HCR_NV2 (UL(1) << 45)#define HCR_AT (UL(1) << 44)#define HCR_NV1 (UL(1) << 43)#define HCR_NV (UL(1) << 42)arch/arm64/include/asm/kvm_emulate.h +13
-9@@ -245,21 +245,24 @@ static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)static inline u64 __fixup_spsr_el2_write (struct kvm_cpu_context *ctxt,u64 val) { -if (!__vcpu_el2_e2h_is_set (ctxt)) { -/* -* Clear the .M field when writing SPSR to the CPU, so that we -* can detect when the CPU clobbered our SPSR copy during a -* local exception. -*/ - val &= ~0xc ; - } + struct kvm_vcpu *vcpu = container_of(ctxt, struct kvm_vcpu, arch.ctxt); + +if (enhanced_nested_virt_in_use (vcpu) ||__vcpu_el2_e2h_is_set (ctxt)) +return val; - return val; +/* +* Clear the .M field when writing SPSR to the CPU, so that we +* can detect when the CPU clobbered our SPSR copy during a +* local exception. +*/ +return val &= ~0xc ; }static inline u64 __fixup_spsr_el2_read (const struct kvm_cpu_context *ctxt,u64 val) { -if (__vcpu_el2_e2h_is_set (ctxt)) + struct kvm_vcpu *vcpu = container_of(ctxt, struct kvm_vcpu, arch.ctxt); + +if (enhanced_nested_virt_in_use (vcpu) ||__vcpu_el2_e2h_is_set (ctxt))return val;/* arch/arm64/include/asm/sysreg.h +1
-0@@ -550,6 +550,7 @@#define SYS_TCR_EL2 sys_reg(3, 4, 2, 0, 2)#define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)#define SYS_VTCR_EL2 sys_reg(3, 4, 2, 1, 2) +#define SYS_VNCR_EL2 sys_reg(3, 4, 2, 2, 0)#define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0)#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1)arch/arm64/kvm/hyp/vhe/switch.c +0
-0arch/arm64/kvm/hyp/vhe/switch.c static void __activate_traps(struct kvm_vcpu *vcpu) · line 49
Quoted 6 lines
arch/arm64/kvm/hyp/vhe/switch.c +2
-1@@ -45,7 +45,13 @@ static void __activate_traps(struct kvm_vcpu *vcpu)* the EL1 virtual memory control register accesses * as well as the AT S1 operations. */ - hcr |=HCR_TVM |HCR_TRVM |HCR_AT |HCR_TTLB |HCR_NV1 ; +if (enhanced_nested_virt_in_use (vcpu)) { + hcr &= ~HCR_TVM ;I think, we should clear TRVM also?
Quoted 1 line
hcr &= ~(HCR_TVM | HCR_TRVM);
Hmmm. But TRVM is never set the first place, is it? It is only here that we augment the host HCR_EL2 with various trap configurations depending on whether the host is NV2 capable or not, whether the guest is VHE or not, and whether the guest as set of additional flags of its own.
Given that, I don't think there is a need to clear this bit.
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Marc Zyngier
maintainerPATCH v5 68/69 KVM: arm64: nv: Fast-track 'InHost' exception returns
In reply to Marc ZyngierA significant part of the ARMv8.3-NV extension is to trap ERET instructions so that the hypervisor gets a chance to switch from a vEL2 L1 guest to an EL1 L2 guest.
But this also has the unfortunate consequence of trapping ERET in unsuspecting circumstances, such as staying at vEL2 (interrupt handling while being in the guest hypervisor), or returning to host userspace in the case of a VHE guest.
Although we already make some effort to handle these ERET quicker by not doing the put/load dance, it is still way too far down the line for it to be efficient enough.
For these cases, it would ideal to ERET directly, no question asked. Of course, we can't do that. But the next best thing is to do it as early as possible, in fixup_guest_exit(), much as we would handle FPSIMD exceptions.
--- arch/arm64/kvm/emulate-nested.c | 26 ++----------------- arch/arm64/kvm/hyp/vhe/switch.c | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 24 deletions(-)
arch/arm64/kvm/emulate-nested.c +2 -24
@@ -52,8 +52,7 @@ bool forward_nv_traps(struct kvm_vcpu *vcpu)void kvm_emulate_nested_eret (struct kvm_vcpu *vcpu) { -u64 spsr, elr, mode; -bool direct_eret; +u64 spsr, elr;/* * Forward this trap to the virtual EL2 if the virtual
@@ -62,31 +61,10 @@ void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu)if (forward_nv_traps (vcpu))return ; -/* -* Going through the whole put/load motions is a waste of time -* if this is a VHE guest hypervisor returning to its own -* userspace, or the hypervisor performing a local exception -* return. No need to save/restore registers, no need to -* switch S2 MMU. Just do the canonical ERET. -*/ - spsr =vcpu_read_sys_reg (vcpu,SPSR_EL2 ); - mode = spsr & (PSR_MODE_MASK |PSR_MODE32_BIT ); - - direct_eret = (mode == PSR_MODE_EL0t && -vcpu_el2_e2h_is_set (vcpu) && -vcpu_el2_tge_is_set (vcpu)); - direct_eret |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t); - -if (direct_eret) { - *vcpu_pc (vcpu) =vcpu_read_sys_reg (vcpu,ELR_EL2 ); - *vcpu_cpsr (vcpu) = spsr; -trace_kvm_nested_eret (vcpu, *vcpu_pc (vcpu), spsr); -return ; - } -preempt_disable ();kvm_arch_vcpu_put (vcpu); + spsr =__vcpu_sys_reg (vcpu,SPSR_EL2 ); elr =__vcpu_sys_reg (vcpu,ELR_EL2 );trace_kvm_nested_eret (vcpu, elr, spsr);
arch/arm64/kvm/hyp/vhe/switch.c +46 -0
58 lines not on this page · raw message
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Marc Zyngier
maintainerPATCH v5 69/69 KVM: arm64: nv: Fast-track EL1 TLBIs for VHE guests
In reply to Marc ZyngierDue to the way ARMv8.4-NV suppresses traps when accessing EL2 system registers, we can't track when the guest changes its HCR_EL2.TGE setting. This means we always trap EL1 TLBIs, even if they don't affect any guest.
This obviously has a huge impact on performance, as we handle TLBI traps as a normal exit, and a normal VHE host issues thousands of TLBIs when booting (and quite a few when running userspace).
A cheap way to reduce the overhead is to handle the limited case of {E2H,TGE}=={1,1} as a guest fixup, as we already have the right mmu configuration in place. Just execute the decoded instruction right away and return to the guest.
--- arch/arm64/kvm/hyp/vhe/switch.c | 43 ++++++++++++++++++++++++++++++++- arch/arm64/kvm/hyp/vhe/tlb.c | 6 +++-- arch/arm64/kvm/sys_regs.c | 25 ++++++------------- 3 files changed, 54 insertions(+), 20 deletions(-)
arch/arm64/kvm/hyp/vhe/switch.c +42 -1
@@ -162,6 +162,47 @@ void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu)__deactivate_traps_common (vcpu); } +static bool kvm_hyp_handle_tlbi_el1 (struct kvm_vcpu *vcpu,u64 *exit_code) +{ +u32 instr; +u64 val; + +/* +* Ideally, we would never trap on EL1 TLB invalidations when the +* guest's HCR_EL2.{E2H,TGE} == {1,1}. But "thanks" to ARMv8.4, we +* don't trap writes to HCR_EL2, meaning that we can't track +* changes to the virtual TGE bit. So we leave HCR_EL2.TTLB set on +* the host. Oopsie... +* +* In order to speed-up EL1 TLBIs from the vEL2 guest when TGE is +* set, try and handle these invalidation as quickly as possible, +* without fully exiting (unless this needs forwarding). +*/ +if (!enhanced_nested_virt_in_use (vcpu) || + !vcpu_mode_el2 (vcpu) || + (__vcpu_sys_reg (vcpu,HCR_EL2 ) & (HCR_E2H |HCR_TGE )) != (HCR_E2H |HCR_TGE )) +return false; + + instr =esr_sys64_to_sysreg (kvm_vcpu_get_esr (vcpu)); +if (sys_reg_Op0 (instr) != TLBI_Op0 || +sys_reg_Op1 (instr) != TLBI_Op1_EL1) +return false; + + val =vcpu_get_reg (vcpu,kvm_vcpu_sys_get_rt (vcpu)); +__kvm_tlb_el1_instr (NULL , val, instr); +__kvm_skip_instr (vcpu); + +return true; +} + +static bool kvm_hyp_handle_sysreg_vhe (struct kvm_vcpu *vcpu,u64 *exit_code) +{ +if (kvm_hyp_handle_tlbi_el1 (vcpu, exit_code)) +return true; + +return kvm_hyp_handle_sysreg (vcpu, exit_code); +} +static bool kvm_hyp_handle_eret (struct kvm_vcpu *vcpu,u64 *exit_code) {struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
@@ -210,7 +251,7 @@ static bool kvm_hyp_handle_eret(struct kvm_vcpu *vcpu, u64 *exit_code)static const exit_handler_fn hyp_exit_handlers[] = { [0 ... ESR_ELx_EC_MAX] =NULL , [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, - [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg, + [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg_vhe, [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd, [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
arch/arm64/kvm/hyp/vhe/tlb.c +4 -2
@@ -201,7 +201,8 @@ void __kvm_tlb_el1_instr(struct kvm_s2_mmu *mmu, u64 val, u64 sys_encoding)dsb (ishst);/* Switch to requested VMID */ -__tlb_switch_to_guest (mmu, &cxt); +if (mmu) +__tlb_switch_to_guest (mmu, &cxt);/* * Execute the same instruction as the guest hypervisor did,
@@ -240,5 +241,6 @@ void __kvm_tlb_el1_instr(struct kvm_s2_mmu *mmu, u64 val, u64 sys_encoding)dsb (ish);isb (); -__tlb_switch_to_host (&cxt); +if (mmu) +__tlb_switch_to_host (&cxt); }
arch/arm64/kvm/sys_regs.c +8 -17
@@ -2804,6 +2804,8 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) {u32 sys_encoding =sys_insn (p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); +u64 virtual_vttbr =vcpu_read_sys_reg (vcpu,VTTBR_EL2 ); +struct kvm_s2_mmu *mmu;/* * If we're here, this is because we've trapped on a EL1 TLBI
@@ -2822,24 +2824,13 @@ static bool handle_tlbi_el1(struct kvm_vcpu *vcpu, struct sys_reg_params *p,mutex_lock (&vcpu->kvm->lock); -if ((__vcpu_sys_reg (vcpu,HCR_EL2 ) & (HCR_E2H |HCR_TGE )) != (HCR_E2H |HCR_TGE )) { -u64 virtual_vttbr =vcpu_read_sys_reg (vcpu,VTTBR_EL2 ); -struct kvm_s2_mmu *mmu; - - mmu =lookup_s2_mmu (vcpu->kvm, virtual_vttbr,HCR_VM ); -if (mmu) -__kvm_tlb_el1_instr (mmu, p->regval, sys_encoding); + mmu =lookup_s2_mmu (vcpu->kvm, virtual_vttbr,HCR_VM ); +if (mmu) +__kvm_tlb_el1_instr (mmu, p->regval, sys_encoding); - mmu =lookup_s2_mmu (vcpu->kvm, virtual_vttbr,0 ); -if (mmu) -__kvm_tlb_el1_instr (mmu, p->regval, sys_encoding); - }else { -/* -* ARMv8.4-NV allows the guest to change TGE behind -* our back, so we always trap EL1 TLBIs from vEL2... -*/ -__kvm_tlb_el1_instr (&vcpu->kvm->arch.mmu, p->regval, sys_encoding); - } + mmu =lookup_s2_mmu (vcpu->kvm, virtual_vttbr,0 ); +if (mmu) +__kvm_tlb_el1_instr (mmu, p->regval, sys_encoding);mutex_unlock (&vcpu->kvm->lock);
Signature · 6 lines
2.30.2 _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: (subset) [PATCH v5 00/69] KVM: arm64: ARMv8.3/8.4 Nested Virtualization support
In reply to Marc ZyngierQuoted 10 lines · Marc Zyngier
Here the bi-annual drop of the KVM/arm64 NV support code, the least loved series in the history of KVM!
Not a lot has changed since [1]: a number of bug fixes (wrong MMU context being selected leading to failing TLB invalidations, fixes around the handling of nested faults), a complete rewrite of the early exit handling, a change in the way the NV support is enabled ("kvm-arm.mode=nested"), and a rebase on top of 5.16-rc1.
[...]
Applied to next, thanks!
[04/69] KVM: arm64: Rework kvm_pgtable initialisation
commit: 9d8604b28575ccab3afd8d6f56cab9a6c0d281ef
Cheers,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.