ARM Nested Virt Support
v4 cover ARM Nested Virt Support
This is a rebase on top of qemu v9.2.0 and tested against Marc's nv-next from Jan 9, 2025. As this branch is continuously evolving I put a snapshot at https://github.com/eauger/linux/tree/nv_next_jan9_2025.
This was tested with rhel L1 guest (4k/64k) and various L2 guests (buildroot, debian, rhel 4k/64k). I used mainline edk2.
The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked with : -machine virt,virtualization=on
This series can be found at:
https://github.com/eauger/qemu/tree/v9.2.0-nv-rfcv4
Previous version from Miguel:
[1] https://lore.kernel.org/all/[email protected]/
Previous version from Haibo:
[2] https://lore.kernel.org/qemu-devel/[email protected]/
---
v3 -> v4:
- fix: only set maint_irq if vms->virt
v2 -> v3:
- KVM EL2 only is set if virtualization option is set
- fixes regression with virtualization=off
- Add checks against unsupported GIC configs until the kernel does
Eric Auger (1):
linux-headers: update to nv-next (Jan 9)
Haibo Xu (4):
hw/arm: Allow setting KVM vGIC maintenance IRQ target/arm/kvm: Add helper to detect EL2 when using KVM target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported hw/arm/virt: Allow virt extensions with KVM
include/hw/intc/arm_gicv3_common.h | 1 + linux-headers/asm-arm64/kvm.h | 1 + linux-headers/linux/kvm.h | 4 +++- target/arm/kvm_arm.h | 12 ++++++++++++ hw/arm/virt.c | 13 ++++++++++++- hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c | 21 +++++++++++++++++++++ target/arm/kvm.c | 21 +++++++++++++++++++++ 8 files changed, 72 insertions(+), 2 deletions(-)
Signature · 1 line
2.47.1
RFC v4 1/5 linux-headers: update to nv-next (Jan 9)
This is a temporary linux header update against Marc's nv-next branch (from Jan 9). As this branch is continuously evolving I made a snapshot at
https://github.com/eauger/linux/tree/nv_next_jan9_2025
--- linux-headers/asm-arm64/kvm.h | 1 + linux-headers/linux/kvm.h | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-)
linux-headers/asm-arm64/kvm.h +1 -0
@@ -397,6 +397,7 @@ enum {#define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 +#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
linux-headers/linux/kvm.h +3 -1
@@ -178,6 +178,7 @@ struct kvm_xen_exit {#define KVM_EXIT_NOTIFY 37#define KVM_EXIT_LOONGARCH_IOCSR 38#define KVM_EXIT_MEMORY_FAULT 39 +#define KVM_EXIT_ARM_LDST64B 40/* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */
@@ -391,7 +392,7 @@ struct kvm_run { } eoi;/* KVM_EXIT_HYPERV */ struct kvm_hyperv_exit hyperv; -/* KVM_EXIT_ARM_NISV */ +/* KVM_EXIT_ARM_NISV / KVM_EXIT_ARM_LDST64B */ struct {__u64 esr_iss;__u64 fault_ipa;
@@ -925,6 +926,7 @@ struct kvm_enable_cap {#define KVM_CAP_PRE_FAULT_MEMORY 236#define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237#define KVM_CAP_X86_GUEST_MODE 238 +#define KVM_CAP_ARM_EL2 239struct kvm_irq_routing_irqchip {__u32 irqchip;
Signature · 1 line
2.47.1
RFC v4 2/5 hw/arm: Allow setting KVM vGIC maintenance IRQ
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
Allow virt arm machine to set the interrupt ID for the KVM GIC maintenance interrupt.
This setting must be done before the KVM_DEV_ARM_VGIC_CTRL_INIT hence the choice to perform the setting in the GICv3 realize instead of proceeding the same way as kvm_arm_pmu_set_irq().
---
v3 -> v4:
- only set maint_irq if vms->virt
v2 -> v3:
- tweak the commit message and explain why we do not proceed the same way as kvm_arm_pmu_set_irq (Peter)
v1 -> v2:
- [Miguel] replaced the has_virt_extensions by the maintenance irq intid property. [Eric] restored kvm_device_check_attr and kvm_device_access standard usage and conditionally call those if the prop is set.
--- include/hw/intc/arm_gicv3_common.h | 1 + hw/arm/virt.c | 3 +++ hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c | 21 +++++++++++++++++++++ 4 files changed, 26 insertions(+)
include/hw/intc/arm_gicv3_common.h +1 -0
@@ -231,6 +231,7 @@ struct GICv3State {uint32_t num_cpu;uint32_t num_irq;uint32_t revision; +uint32_t maint_irq;bool lpi_enable;bool nmi_support;bool security_extn;
hw/arm/virt.c +3 -0
@@ -827,6 +827,9 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) OBJECT(mem), &error_fatal);qdev_prop_set_bit (vms->gic,"has-lpi" , true); } + }else if (vms->virt) { +qdev_prop_set_uint32 (vms->gic,"maintenance-interrupt-id" , +ARCH_GIC_MAINT_IRQ ); } }else {if (!kvm_irqchip_in_kernel ()) {
hw/intc/arm_gicv3_common.c +1 -0
@@ -612,6 +612,7 @@ static const Property arm_gicv3_common_properties[] = {DEFINE_PROP_BOOL ("has-lpi" , GICv3State, lpi_enable,0 ),DEFINE_PROP_BOOL ("has-nmi" , GICv3State, nmi_support,0 ),DEFINE_PROP_BOOL ("has-security-extensions" , GICv3State, security_extn,0 ), +DEFINE_PROP_UINT32 ("maintenance-interrupt-id" , GICv3State, maint_irq,0 ),/* * Compatibility property: force 8 bits of physical priority, even * if the CPU being emulated should have fewer.
hw/intc/arm_gicv3_kvm.c +21 -0
@@ -22,6 +22,7 @@#include "qemu/osdep.h" #include "qapi/error.h" #include "hw/intc/arm_gicv3_common.h" +#include "hw/arm/virt.h" #include "qemu/error-report.h" #include "qemu/module.h" #include "system/kvm.h"
@@ -825,6 +826,26 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)return ; } +if (s->maint_irq) { +int ret; + + ret =kvm_device_check_attr (s->dev_fd, +KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ ,0 ); +if (!ret) { +error_setg_errno (errp, errno, +"VGICv3 setting maintenance IRQ is not " +"supported by this host kernel" ); +return ; + } + + ret =kvm_device_access (s->dev_fd,KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ ,0 , + &s->maint_irq, true, errp); +if (ret) { +error_setg_errno (errp, errno,"Failed to set VGIC maintenance IRQ" ); +return ; + } + } + multiple_redist_region_allowed =kvm_device_check_attr (s->dev_fd,KVM_DEV_ARM_VGIC_GRP_ADDR ,KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION );
Signature · 1 line
2.47.1
RFC v4 3/5 target/arm/kvm: Add helper to detect EL2 when using KVM
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
Introduce query support for KVM_CAP_ARM_EL2.
--- target/arm/kvm_arm.h | 12 ++++++++++++ target/arm/kvm.c | 5 +++++ 2 files changed, 17 insertions(+)
target/arm/kvm_arm.h +12 -0
@@ -193,6 +193,13 @@ bool kvm_arm_sve_supported(void);*/ bool kvm_arm_mte_supported (void ); +/** +* kvm_arm_el2_supported: +* +* Returns true if KVM can enable EL2 and false otherwise. +*/ +bool kvm_arm_el2_supported (void ); +/** * kvm_arm_get_max_vm_ipa_size: * @ms: Machine state handle
@@ -247,6 +254,11 @@ static inline bool kvm_arm_mte_supported(void)return false; } +static inline bool kvm_arm_el2_supported (void ) +{ +return false; +} +/* * These functions should never actually be called without KVM support. */
target/arm/kvm.c +5 -0
@@ -1797,6 +1797,11 @@ bool kvm_arm_aarch32_supported(void)return kvm_check_extension (kvm_state,KVM_CAP_ARM_EL1_32BIT ); } +bool kvm_arm_el2_supported (void ) +{ +return kvm_check_extension (kvm_state,KVM_CAP_ARM_EL2 ); +} +bool kvm_arm_sve_supported (void ) {return kvm_check_extension (kvm_state,KVM_CAP_ARM_SVE );
Signature · 1 line
2.47.1
RFC v4 4/5 target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
KVM_CAP_ARM_EL2 must be supported by the cpu to enable ARM_FEATURE_EL2. In case the host does support NV, expose the feature.
---
v2 -> v3:
- check pmu->has_el2 on kvm_arch_init_vcpu() when setting KVM_ARM_VCPU_HAS_EL2 feature (Peter)
v1 -> v2:
- remove isar_feature_aa64_aa32_el2 modif in target/arm/cpu.h [Richard] and use el2_supported in kvm_arch_init_vcpu
--- target/arm/kvm.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
target/arm/kvm.c +16 -0
@@ -255,6 +255,7 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)*/ int fdarray[3 ];bool sve_supported; +bool el2_supported;bool pmu_supported = false;uint64_t features =0 ;int err;
@@ -285,6 +286,14 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) init.features[0 ] |=1 <<KVM_ARM_VCPU_SVE ; } +/* +* Ask for EL2 if supported. +*/ + el2_supported =kvm_arm_el2_supported (); +if (el2_supported) { + init.features[0 ] |=1 <<KVM_ARM_VCPU_HAS_EL2 ; + } +/* * Ask for Pointer Authentication if supported, so that we get * the unsanitized field values for AA64ISAR1_EL1.
@@ -468,6 +477,10 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) features |=1ULL <<ARM_FEATURE_AARCH64 ; features |=1ULL <<ARM_FEATURE_GENERIC_TIMER ; +if (el2_supported) { + features |=1ULL <<ARM_FEATURE_EL2 ; + } + ahcf->features = features;return true;
@@ -1918,6 +1931,9 @@ int kvm_arch_init_vcpu(CPUState *cs) cpu->kvm_init_features[0 ] |= (1 <<KVM_ARM_VCPU_PTRAUTH_ADDRESS |1 <<KVM_ARM_VCPU_PTRAUTH_GENERIC ); } +if (cpu->has_el2 &&kvm_arm_el2_supported ()) { + cpu->kvm_init_features[0 ] |=1 <<KVM_ARM_VCPU_HAS_EL2 ; + }/* Do KVM_ARM_VCPU_INIT ioctl */ ret =kvm_arm_vcpu_init (cpu);
Signature · 1 line
2.47.1
RFC v4 5/5 hw/arm/virt: Allow virt extensions with KVM
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
---
v2 -> v3:
- check gic version/in-kernel implementation when kvm el2 is set (Peter)
v1 -> v2:
- fixed test ordering: virt && ((kvm && !kvm_el2) || hvf) [Richard]
- tweeked the commit title & message
--- hw/arm/virt.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
hw/arm/virt.c +9 -1
@@ -791,6 +791,13 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)default :g_assert_not_reached (); } + +if (kvm_enabled () && vms->virt && + (revision !=3 || !kvm_irqchip_in_kernel ())) { +error_report ("KVM EL2 only is supported with in-kernel GICv3" ); +exit (1 ); + } + vms->gic =qdev_new (gictype);qdev_prop_set_uint32 (vms->gic,"revision" , revision);qdev_prop_set_uint32 (vms->gic,"num-cpu" , smp_cpus);
@@ -2210,7 +2217,8 @@ static void machvirt_init(MachineState *machine)exit (1 ); } -if (vms->virt && (kvm_enabled () ||hvf_enabled ())) { +if (vms->virt && + ((kvm_enabled () && !kvm_arm_el2_supported ()) ||hvf_enabled ())) {error_report ("mach-virt: %s does not support providing " "Virtualization extensions to the guest CPU" ,current_accel_name ());
Signature · 1 line
2.47.1