Skip to contentSkip to conversation

ARM Nested Virt Support

RFC Superseded Eric Auger 1 participant 6 messages Jan 16, 2025

v4 of 9 Superseded Latest is v9 →

lore
Share & tools
lore

Eric Auger

author
#0 lore raw

v4 cover ARM Nested Virt Support

This is a rebase on top of qemu v9.2.0 and tested against Marc's nv-next from Jan 9, 2025. As this branch is continuously evolving I put a snapshot at https://github.com/eauger/linux/tree/nv_next_jan9_2025.

This was tested with rhel L1 guest (4k/64k) and various L2 guests (buildroot, debian, rhel 4k/64k). I used mainline edk2.

The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked with : -machine virt,virtualization=on

This series can be found at:
https://github.com/eauger/qemu/tree/v9.2.0-nv-rfcv4

Previous version from Miguel:
[1] https://lore.kernel.org/all/[email protected]/
Previous version from Haibo:
[2] https://lore.kernel.org/qemu-devel/[email protected]/

---
v3 -> v4:
- fix: only set maint_irq if vms->virt

v2 -> v3:
- KVM EL2 only is set if virtualization option is set
- fixes regression with virtualization=off
- Add checks against unsupported GIC configs until the kernel does

Eric Auger (1):
linux-headers: update to nv-next (Jan 9)

Haibo Xu (4):
hw/arm: Allow setting KVM vGIC maintenance IRQ target/arm/kvm: Add helper to detect EL2 when using KVM target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported hw/arm/virt: Allow virt extensions with KVM

 include/hw/intc/arm_gicv3_common.h |  1 +
 linux-headers/asm-arm64/kvm.h      |  1 +
 linux-headers/linux/kvm.h          |  4 +++-
 target/arm/kvm_arm.h               | 12 ++++++++++++
 hw/arm/virt.c                      | 13 ++++++++++++-
 hw/intc/arm_gicv3_common.c         |  1 +
 hw/intc/arm_gicv3_kvm.c            | 21 +++++++++++++++++++++
 target/arm/kvm.c                   | 21 +++++++++++++++++++++
 8 files changed, 72 insertions(+), 2 deletions(-)
Signature · 1 line
2.47.1
author
#1 lore raw

RFC v4 1/5 linux-headers: update to nv-next (Jan 9)

This is a temporary linux header update against Marc's nv-next branch (from Jan 9). As this branch is continuously evolving I made a snapshot at
https://github.com/eauger/linux/tree/nv_next_jan9_2025

---
 linux-headers/asm-arm64/kvm.h | 1 +
 linux-headers/linux/kvm.h     | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)
linux-headers/asm-arm64/kvm.h +1 -0
index dccd5d965f..b81e7d81a1 100644
@@ -397,6 +397,7 @@ enum {
 #define KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS 6
 #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO  7
 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8
+#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ  9
 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT	10
 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \
 			(0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)
linux-headers/linux/kvm.h +3 -1
index 3bcd4eabe3..1c8431ff43 100644
@@ -178,6 +178,7 @@ struct kvm_xen_exit {
 #define KVM_EXIT_NOTIFY           37
 #define KVM_EXIT_LOONGARCH_IOCSR  38
 #define KVM_EXIT_MEMORY_FAULT     39
+#define KVM_EXIT_ARM_LDST64B      40
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -391,7 +392,7 @@ struct kvm_run {
 		} eoi;
 		/* KVM_EXIT_HYPERV */
 		struct kvm_hyperv_exit hyperv;
-		/* KVM_EXIT_ARM_NISV */
+		/* KVM_EXIT_ARM_NISV / KVM_EXIT_ARM_LDST64B */
 		struct {
 			__u64 esr_iss;
 			__u64 fault_ipa;
@@ -925,6 +926,7 @@ struct kvm_enable_cap {
 #define KVM_CAP_PRE_FAULT_MEMORY 236
 #define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237
 #define KVM_CAP_X86_GUEST_MODE 238
+#define KVM_CAP_ARM_EL2 239
 
 struct kvm_irq_routing_irqchip {
 	__u32 irqchip;
Signature · 1 line
2.47.1
author
#2 lore raw

RFC v4 2/5 hw/arm: Allow setting KVM vGIC maintenance IRQ

In reply to Eric Auger

From: Haibo Xu <[email protected]>

Allow virt arm machine to set the interrupt ID for the KVM GIC maintenance interrupt.

This setting must be done before the KVM_DEV_ARM_VGIC_CTRL_INIT hence the choice to perform the setting in the GICv3 realize instead of proceeding the same way as kvm_arm_pmu_set_irq().

---

v3 -> v4:
- only set maint_irq if vms->virt

v2 -> v3:
- tweak the commit message and explain why we do not proceed the same way as kvm_arm_pmu_set_irq (Peter)

v1 -> v2:
- [Miguel] replaced the has_virt_extensions by the maintenance irq intid property. [Eric] restored kvm_device_check_attr and kvm_device_access standard usage and conditionally call those if the prop is set.

---
 include/hw/intc/arm_gicv3_common.h |  1 +
 hw/arm/virt.c                      |  3 +++
 hw/intc/arm_gicv3_common.c         |  1 +
 hw/intc/arm_gicv3_kvm.c            | 21 +++++++++++++++++++++
 4 files changed, 26 insertions(+)
include/hw/intc/arm_gicv3_common.h +1 -0
index a3d6a0e507..c18503869f 100644
@@ -231,6 +231,7 @@ struct GICv3State {
     uint32_t num_cpu;
     uint32_t num_irq;
     uint32_t revision;
+    uint32_t maint_irq;
     bool lpi_enable;
     bool nmi_support;
     bool security_extn;
hw/arm/virt.c +3 -0
index 99e0a68b6c..55b0bb24ba 100644
@@ -827,6 +827,9 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
                                          OBJECT(mem), &error_fatal);
                 qdev_prop_set_bit(vms->gic, "has-lpi", true);
             }
+        } else if (vms->virt) {
+            qdev_prop_set_uint32(vms->gic, "maintenance-interrupt-id",
+                                 ARCH_GIC_MAINT_IRQ);
         }
     } else {
         if (!kvm_irqchip_in_kernel()) {
hw/intc/arm_gicv3_common.c +1 -0
index 76b2283c92..2ed50d7d07 100644
@@ -612,6 +612,7 @@ static const Property arm_gicv3_common_properties[] = {
     DEFINE_PROP_BOOL("has-lpi", GICv3State, lpi_enable, 0),
     DEFINE_PROP_BOOL("has-nmi", GICv3State, nmi_support, 0),
     DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+    DEFINE_PROP_UINT32("maintenance-interrupt-id", GICv3State, maint_irq, 0),
     /*
      * Compatibility property: force 8 bits of physical priority, even
      * if the CPU being emulated should have fewer.
hw/intc/arm_gicv3_kvm.c +21 -0
index 8e17cab2a0..7248055a02 100644
@@ -22,6 +22,7 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "hw/intc/arm_gicv3_common.h"
+#include "hw/arm/virt.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "system/kvm.h"
@@ -825,6 +826,26 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
         return;
     }
 
+    if (s->maint_irq) {
+        int ret;
+
+        ret = kvm_device_check_attr(s->dev_fd,
+                                    KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0);
+        if (!ret) {
+            error_setg_errno(errp, errno,
+                             "VGICv3 setting maintenance IRQ is not "
+                             "supported by this host kernel");
+            return;
+        }
+
+        ret = kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0,
+                                &s->maint_irq, true, errp);
+        if (ret) {
+            error_setg_errno(errp, errno, "Failed to set VGIC maintenance IRQ");
+            return;
+       }
+    }
+
     multiple_redist_region_allowed =
         kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_ADDR,
                               KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION);
Signature · 1 line
2.47.1
author
#3 lore raw

RFC v4 3/5 target/arm/kvm: Add helper to detect EL2 when using KVM

In reply to Eric Auger

From: Haibo Xu <[email protected]>

Introduce query support for KVM_CAP_ARM_EL2.

---
 target/arm/kvm_arm.h | 12 ++++++++++++
 target/arm/kvm.c     |  5 +++++
 2 files changed, 17 insertions(+)
target/arm/kvm_arm.h +12 -0
index 05c3de8cd4..8c1314da20 100644
@@ -193,6 +193,13 @@ bool kvm_arm_sve_supported(void);
  */
 bool kvm_arm_mte_supported(void);
 
+/**
+ * kvm_arm_el2_supported:
+ *
+ * Returns true if KVM can enable EL2 and false otherwise.
+ */
+bool kvm_arm_el2_supported(void);
+
 /**
  * kvm_arm_get_max_vm_ipa_size:
  * @ms: Machine state handle
@@ -247,6 +254,11 @@ static inline bool kvm_arm_mte_supported(void)
     return false;
 }
 
+static inline bool kvm_arm_el2_supported(void)
+{
+    return false;
+}
+
 /*
  * These functions should never actually be called without KVM support.
  */
target/arm/kvm.c +5 -0
index a9444a2c7a..43362164eb 100644
@@ -1797,6 +1797,11 @@ bool kvm_arm_aarch32_supported(void)
     return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT);
 }
 
+bool kvm_arm_el2_supported(void)
+{
+    return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL2);
+}
+
 bool kvm_arm_sve_supported(void)
 {
     return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE);
Signature · 1 line
2.47.1
author
#4 lore raw

RFC v4 4/5 target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported

In reply to Eric Auger

From: Haibo Xu <[email protected]>

KVM_CAP_ARM_EL2 must be supported by the cpu to enable ARM_FEATURE_EL2. In case the host does support NV, expose the feature.

---
v2 -> v3:
- check pmu->has_el2 on kvm_arch_init_vcpu() when setting KVM_ARM_VCPU_HAS_EL2 feature (Peter)

v1 -> v2:
- remove isar_feature_aa64_aa32_el2 modif in target/arm/cpu.h [Richard] and use el2_supported in kvm_arch_init_vcpu

---
 target/arm/kvm.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
target/arm/kvm.c +16 -0
index 43362164eb..bfaf48824c 100644
@@ -255,6 +255,7 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
      */
     int fdarray[3];
     bool sve_supported;
+    bool el2_supported;
     bool pmu_supported = false;
     uint64_t features = 0;
     int err;
@@ -285,6 +286,14 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
         init.features[0] |= 1 << KVM_ARM_VCPU_SVE;
     }
 
+    /*
+     * Ask for EL2 if supported.
+     */
+    el2_supported = kvm_arm_el2_supported();
+    if (el2_supported) {
+        init.features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2;
+    }
+
     /*
      * Ask for Pointer Authentication if supported, so that we get
      * the unsanitized field values for AA64ISAR1_EL1.
@@ -468,6 +477,10 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
     features |= 1ULL << ARM_FEATURE_AARCH64;
     features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
 
+    if (el2_supported) {
+        features |= 1ULL << ARM_FEATURE_EL2;
+    }
+
     ahcf->features = features;
 
     return true;
@@ -1918,6 +1931,9 @@ int kvm_arch_init_vcpu(CPUState *cs)
         cpu->kvm_init_features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS |
                                       1 << KVM_ARM_VCPU_PTRAUTH_GENERIC);
     }
+    if (cpu->has_el2 && kvm_arm_el2_supported()) {
+        cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2;
+    }
 
     /* Do KVM_ARM_VCPU_INIT ioctl */
     ret = kvm_arm_vcpu_init(cpu);
Signature · 1 line
2.47.1
author
#5 lore raw

RFC v4 5/5 hw/arm/virt: Allow virt extensions with KVM

In reply to Eric Auger

From: Haibo Xu <[email protected]>

Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.

Also check only in-kernel GICv3 is used along with KVM EL2.

---
v2 -> v3:
- check gic version/in-kernel implementation when kvm el2 is set (Peter)

v1 -> v2:
- fixed test ordering: virt && ((kvm && !kvm_el2) || hvf) [Richard]
- tweeked the commit title & message

---
 hw/arm/virt.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
hw/arm/virt.c +9 -1
index 55b0bb24ba..fff25dd615 100644
@@ -791,6 +791,13 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
     default:
         g_assert_not_reached();
     }
+
+    if (kvm_enabled() && vms->virt &&
+        (revision != 3 || !kvm_irqchip_in_kernel())) {
+        error_report("KVM EL2 only is supported with in-kernel GICv3");
+        exit(1);
+    }
+
     vms->gic = qdev_new(gictype);
     qdev_prop_set_uint32(vms->gic, "revision", revision);
     qdev_prop_set_uint32(vms->gic, "num-cpu", smp_cpus);
@@ -2210,7 +2217,8 @@ static void machvirt_init(MachineState *machine)
         exit(1);
     }
 
-    if (vms->virt && (kvm_enabled() || hvf_enabled())) {
+    if (vms->virt &&
+        ((kvm_enabled() && !kvm_arm_el2_supported()) || hvf_enabled())) {
         error_report("mach-virt: %s does not support providing "
                      "Virtualization extensions to the guest CPU",
                      current_accel_name());
Signature · 1 line
2.47.1