Skip to contentSkip to conversation

ARM Nested Virt Support

PATCH Applied Eric Auger 3 participants 12 messages Jul 7, 2025 – Jul 10, 2025

lore
Share & tools
lore

Eric Auger

author
#0 lore raw

v9 cover ARM Nested Virt Support

This is candidate for 10.1.

For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.

This series can be found at:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v9
previous:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v8

Original version from Miguel:
[1] https://lore.kernel.org/all/[email protected]/
version from Haibo:
[2] https://lore.kernel.org/qemu-devel/[email protected]/

History:
v8 -> v9:
- Move the migration blocker to the GIC code

v7 -> v8
- add migration blocker when virt is set along with kvm enabled
- test virt is not set with -cpu host,aarch64=off
- s/only is/is only

v6 -> v7:
- rebase on Peter's target-arm.next. as a result linux header update could be dropped. Faced a small conflict in hw/arm/virt: Allow virt extensions with KVM due to recently landed hw/arm/virt: Make EL2 accelerator check an accept-list. I dared to keep Richard's R-b though.

v5 -> v6:
- linux header update against v6.16-rc2

v4 -> v5:
- rebase on top of v10.0.0

v3 -> v4:
- fix: only set maint_irq if vms->virt

v2 -> v3:
- KVM EL2 only is set if virtualization option is set
- fixes regression with virtualization=off
- Add checks against unsupported GIC configs until the kernel does

Eric Auger (1):
hw/arm/arm_gicv3_kvm: Add a migration blocker with kvm nested virt

Haibo Xu (4):
hw/arm: Allow setting KVM vGIC maintenance IRQ target/arm/kvm: Add helper to detect EL2 when using KVM target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported hw/arm/virt: Allow virt extensions with KVM

 include/hw/intc/arm_gicv3_common.h |  1 +
 target/arm/kvm_arm.h               |  7 +++++++
 hw/arm/virt.c                      | 17 ++++++++++++++++-
 hw/intc/arm_gicv3_common.c         |  1 +
 hw/intc/arm_gicv3_kvm.c            | 29 +++++++++++++++++++++++++++++
 target/arm/kvm-stub.c              |  5 +++++
 target/arm/kvm.c                   | 21 +++++++++++++++++++++
 7 files changed, 80 insertions(+), 1 deletion(-)
Signature · 1 line
2.49.0
author
#1 lore raw

PATCH v9 1/5 hw/arm: Allow setting KVM vGIC maintenance IRQ

From: Haibo Xu <[email protected]>

Allow virt arm machine to set the interrupt ID for the KVM GIC maintenance interrupt.

This setting must be done before the KVM_DEV_ARM_VGIC_CTRL_INIT hence the choice to perform the setting in the GICv3 realize instead of proceeding the same way as kvm_arm_pmu_set_irq().

---

v3 -> v4:
- only set maint_irq if vms->virt

v2 -> v3:
- tweak the commit message and explain why we do not proceed the same way as kvm_arm_pmu_set_irq (Peter)

v1 -> v2:
- [Miguel] replaced the has_virt_extensions by the maintenance irq intid property. [Eric] restored kvm_device_check_attr and kvm_device_access standard usage and conditionally call those if the prop is set.

---
 include/hw/intc/arm_gicv3_common.h |  1 +
 hw/arm/virt.c                      |  3 +++
 hw/intc/arm_gicv3_common.c         |  1 +
 hw/intc/arm_gicv3_kvm.c            | 21 +++++++++++++++++++++
 4 files changed, 26 insertions(+)
include/hw/intc/arm_gicv3_common.h +1 -0
index a3d6a0e507..c18503869f 100644
@@ -231,6 +231,7 @@ struct GICv3State {
     uint32_t num_cpu;
     uint32_t num_irq;
     uint32_t revision;
+    uint32_t maint_irq;
     bool lpi_enable;
     bool nmi_support;
     bool security_extn;
hw/arm/virt.c +3 -0
index 3bcdf92e2f..550a272fbb 100644
@@ -828,6 +828,9 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
                                          OBJECT(mem), &error_fatal);
                 qdev_prop_set_bit(vms->gic, "has-lpi", true);
             }
+        } else if (vms->virt) {
+            qdev_prop_set_uint32(vms->gic, "maintenance-interrupt-id",
+                                 ARCH_GIC_MAINT_IRQ);
         }
     } else {
         if (!kvm_irqchip_in_kernel()) {
hw/intc/arm_gicv3_common.c +1 -0
index 1cee68193c..e438d8c042 100644
@@ -612,6 +612,7 @@ static const Property arm_gicv3_common_properties[] = {
     DEFINE_PROP_BOOL("has-lpi", GICv3State, lpi_enable, 0),
     DEFINE_PROP_BOOL("has-nmi", GICv3State, nmi_support, 0),
     DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+    DEFINE_PROP_UINT32("maintenance-interrupt-id", GICv3State, maint_irq, 0),
     /*
      * Compatibility property: force 8 bits of physical priority, even
      * if the CPU being emulated should have fewer.
hw/intc/arm_gicv3_kvm.c +21 -0
index 3be3bf6c28..b30aac7aee 100644
@@ -22,6 +22,7 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "hw/intc/arm_gicv3_common.h"
+#include "hw/arm/virt.h"
 #include "qemu/error-report.h"
 #include "qemu/module.h"
 #include "system/kvm.h"
@@ -825,6 +826,26 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
         return;
     }
 
+    if (s->maint_irq) {
+        int ret;
+
+        ret = kvm_device_check_attr(s->dev_fd,
+                                    KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0);
+        if (!ret) {
+            error_setg_errno(errp, errno,
+                             "VGICv3 setting maintenance IRQ is not "
+                             "supported by this host kernel");
+            return;
+        }
+
+        ret = kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0,
+                                &s->maint_irq, true, errp);
+        if (ret) {
+            error_setg_errno(errp, errno, "Failed to set VGIC maintenance IRQ");
+            return;
+       }
+    }
+
     multiple_redist_region_allowed =
         kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_ADDR,
                               KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION);
Signature · 1 line
2.49.0
author
#2 lore raw

PATCH v9 2/5 target/arm/kvm: Add helper to detect EL2 when using KVM

In reply to Eric Auger

From: Haibo Xu <[email protected]>

Introduce query support for KVM_CAP_ARM_EL2.

---
 target/arm/kvm_arm.h  | 7 +++++++
 target/arm/kvm-stub.c | 5 +++++
 target/arm/kvm.c      | 5 +++++
 3 files changed, 17 insertions(+)
target/arm/kvm_arm.h +7 -0
index 7dc83caed5..b4cad05155 100644
@@ -191,6 +191,13 @@ bool kvm_arm_sve_supported(void);
  */
 bool kvm_arm_mte_supported(void);
 
+/**
+ * kvm_arm_el2_supported:
+ *
+ * Returns true if KVM can enable EL2 and false otherwise.
+ */
+bool kvm_arm_el2_supported(void);
+
 /**
  * kvm_arm_get_max_vm_ipa_size:
  * @ms: Machine state handle
target/arm/kvm-stub.c +5 -0
index 34e57fab01..c93462c5b9 100644
@@ -47,6 +47,11 @@ bool kvm_arm_mte_supported(void)
     return false;
 }
 
+bool kvm_arm_el2_supported(void)
+{
+    return false;
+}
+
 /*
  * These functions should never actually be called without KVM support.
  */
target/arm/kvm.c +5 -0
index 426f8b159e..e5708e54ae 100644
@@ -1762,6 +1762,11 @@ bool kvm_arm_aarch32_supported(void)
     return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT);
 }
 
+bool kvm_arm_el2_supported(void)
+{
+    return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL2);
+}
+
 bool kvm_arm_sve_supported(void)
 {
     return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE);
Signature · 1 line
2.49.0
author
#3 lore raw

PATCH v9 3/5 target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported

In reply to Eric Auger

From: Haibo Xu <[email protected]>

KVM_CAP_ARM_EL2 must be supported by the cpu to enable ARM_FEATURE_EL2. In case the host does support NV, expose the feature.

---
v2 -> v3:
- check pmu->has_el2 on kvm_arch_init_vcpu() when setting KVM_ARM_VCPU_HAS_EL2 feature (Peter)

v1 -> v2:
- remove isar_feature_aa64_aa32_el2 modif in target/arm/cpu.h [Richard] and use el2_supported in kvm_arch_init_vcpu

---
 target/arm/kvm.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
target/arm/kvm.c +16 -0
index e5708e54ae..46e5f19637 100644
@@ -250,6 +250,7 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
      */
     int fdarray[3];
     bool sve_supported;
+    bool el2_supported;
     bool pmu_supported = false;
     uint64_t features = 0;
     int err;
@@ -269,6 +270,14 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
         init.features[0] |= 1 << KVM_ARM_VCPU_SVE;
     }
 
+    /*
+     * Ask for EL2 if supported.
+     */
+    el2_supported = kvm_arm_el2_supported();
+    if (el2_supported) {
+        init.features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2;
+    }
+
     /*
      * Ask for Pointer Authentication if supported, so that we get
      * the unsanitized field values for AA64ISAR1_EL1.
@@ -422,6 +431,10 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
     features |= 1ULL << ARM_FEATURE_AARCH64;
     features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
 
+    if (el2_supported) {
+        features |= 1ULL << ARM_FEATURE_EL2;
+    }
+
     ahcf->features = features;
 
     return true;
@@ -1887,6 +1900,9 @@ int kvm_arch_init_vcpu(CPUState *cs)
         cpu->kvm_init_features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS |
                                       1 << KVM_ARM_VCPU_PTRAUTH_GENERIC);
     }
+    if (cpu->has_el2 && kvm_arm_el2_supported()) {
+        cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_HAS_EL2;
+    }
 
     /* Do KVM_ARM_VCPU_INIT ioctl */
     ret = kvm_arm_vcpu_init(cpu);
Signature · 1 line
2.49.0
author
#4 lore raw

PATCH v9 4/5 hw/arm/arm_gicv3_kvm: Add a migration blocker with kvm nested virt

In reply to Eric Auger

We may be miss some NV related GIC register save/restore. Until we complete the study, let's add a migration blocker when the maintenance IRQ is set.

---
 hw/intc/arm_gicv3_kvm.c | 8 ++++++++
 1 file changed, 8 insertions(+)
hw/intc/arm_gicv3_kvm.c +8 -0
index b30aac7aee..fccb3886bf 100644
@@ -827,8 +827,16 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
     }
 
     if (s->maint_irq) {
+        Error *kvm_nv_migration_blocker = NULL;
         int ret;
 
+        error_setg(&kvm_nv_migration_blocker,
+                       "Live migration disabled due to KVM nested virt enabled");
+        if (migrate_add_blocker(&kvm_nv_migration_blocker, errp)) {
+            error_free(kvm_nv_migration_blocker);
+            return;
+        }
+
         ret = kvm_device_check_attr(s->dev_fd,
                                     KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0);
         if (!ret) {
Signature · 1 line
2.49.0
#5 lore raw

Re: [PATCH v9 4/5] hw/arm/arm_gicv3_kvm: Add a migration blocker with kvm nested virt

Quoted 15 lines · Eric Auger

We may be miss some NV related GIC register save/restore. Until we complete the study, let's add a migration blocker when the maintenance IRQ is set.

---
  hw/intc/arm_gicv3_kvm.c | 8 ++++++++
  1 file changed, 8 insertions(+)
hw/intc/arm_gicv3_kvm.c +0 -0
index b30aac7aee..fccb3886bf 100644

hw/intc/arm_gicv3_kvm.c static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp) · line 834

Quoted 8 lines
hw/intc/arm_gicv3_kvm.c +3 -0
@@ -827,8 +827,16 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
      }
 
      if (s->maint_irq) {
+        Error *kvm_nv_migration_blocker = NULL;
          int ret;
 
+        error_setg(&kvm_nv_migration_blocker,
+                       "Live migration disabled due to KVM nested virt enabled");

(mis-indentation)

Quoted 8 lines
hw/intc/arm_gicv3_kvm.c +5 -0
+        if (migrate_add_blocker(&kvm_nv_migration_blocker, errp)) {
+            error_free(kvm_nv_migration_blocker);
+            return;
+        }
+
          ret = kvm_device_check_attr(s->dev_fd,
                                      KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ, 0);
          if (!ret) {
author
#6 lore raw

Re: [PATCH v9 4/5] hw/arm/arm_gicv3_kvm: Add a migration blocker with kvm nested virt

Quoted 11 lines · Philippe Mathieu-Daudé
Quoted 10 lines · Eric Auger

We may be miss some NV related GIC register save/restore. Until we complete the study, let's add a migration blocker when the maintenance IRQ is set.

---
  hw/intc/arm_gicv3_kvm.c | 8 ++++++++

  1 file changed, 8 insertions(+)
Quoted 15 lines

hw/intc/arm_gicv3_kvm.c static void kvm_arm_gicv3_realize(DeviceState · line 834

Quoted 13 lines
hw/intc/arm_gicv3_kvm.c +2 -0
index b30aac7aee..fccb3886bf 100644
@@ -827,8 +827,16 @@ static void kvm_arm_gicv3_realize(DeviceState
 *dev, Error **errp)
       }
         if (s->maint_irq) {
+        Error *kvm_nv_migration_blocker = NULL;
           int ret;
   +        error_setg(&kvm_nv_migration_blocker,
+                       "Live migration disabled due to KVM nested
 virt enabled");

(mis-indentation)

Did not notice as checkpatch does not complain.

Eric

Quoted 11 lines
Quoted 10 lines
hw/intc/arm_gicv3_kvm.c +5 -0
+        if (migrate_add_blocker(&kvm_nv_migration_blocker, errp)) {
+            error_free(kvm_nv_migration_blocker);
+            return;
+        }
+
           ret = kvm_device_check_attr(s->dev_fd,
                                       KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ,

0);
          if (!ret) {

#7 lore raw

Re: [PATCH v9 4/5] hw/arm/arm_gicv3_kvm: Add a migration blocker with kvm nested virt

Quoted 29 lines · Eric Auger
Quoted 11 lines · Philippe Mathieu-Daudé
Quoted 10 lines · Eric Auger

We may be miss some NV related GIC register save/restore. Until we complete the study, let's add a migration blocker when the maintenance IRQ is set.

---
   hw/intc/arm_gicv3_kvm.c | 8 ++++++++
   1 file changed, 8 insertions(+)
Quoted 15 lines

hw/intc/arm_gicv3_kvm.c static void kvm_arm_gicv3_realize(DeviceState · line 834

Quoted 13 lines
hw/intc/arm_gicv3_kvm.c +2 -0
index b30aac7aee..fccb3886bf 100644
@@ -827,8 +827,16 @@ static void kvm_arm_gicv3_realize(DeviceState
 *dev, Error **errp)
       }
         if (s->maint_irq) {
+        Error *kvm_nv_migration_blocker = NULL;
           int ret;
   +        error_setg(&kvm_nv_migration_blocker,
+                       "Live migration disabled due to KVM nested
 virt enabled");

(mis-indentation)

Quoted 1 line

Did not notice as checkpatch does not complain.

No worry, it is a simple nudge to Peter for possible maintainer fixup.

#8 lore raw

Re: [PATCH v9 4/5] hw/arm/arm_gicv3_kvm: Add a migration blocker with kvm nested virt

In reply to Eric Auger
Quoted 30 lines · Eric Auger
Quoted 11 lines · Philippe Mathieu-Daudé
Quoted 10 lines · Eric Auger

We may be miss some NV related GIC register save/restore. Until we complete the study, let's add a migration blocker when the maintenance IRQ is set.

---
  hw/intc/arm_gicv3_kvm.c | 8 ++++++++
  1 file changed, 8 insertions(+)
Quoted 15 lines

hw/intc/arm_gicv3_kvm.c static void kvm_arm_gicv3_realize(DeviceState · line 834

Quoted 13 lines
hw/intc/arm_gicv3_kvm.c +2 -0
index b30aac7aee..fccb3886bf 100644
@@ -827,8 +827,16 @@ static void kvm_arm_gicv3_realize(DeviceState
 *dev, Error **errp)
      }
        if (s->maint_irq) {
+        Error *kvm_nv_migration_blocker = NULL;
          int ret;
  +        error_setg(&kvm_nv_migration_blocker,
+                       "Live migration disabled due to KVM nested
 virt enabled");

(mis-indentation)

Quoted 1 line

Did not notice as checkpatch does not complain.

I'll fix the indent in target-arm.next.

thanks
-- PMM

author
#9 lore raw

PATCH v9 5/5 hw/arm/virt: Allow virt extensions with KVM

In reply to Eric Auger

From: Haibo Xu <[email protected]>

Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.

Check neither in-kernel GICv3 nor aarch64=off is used along with KVM EL2.

---
v8 -> v9:
- remove the migration blocker from the machine code as it is now in the GIC code itself.

v7 -> v8:
- add the migration blocker
- s/only is/is only
- check aarch64=off
- update the commit message

v6 -> v7:
- rebase on top of "hw/arm/virt: Make EL2 accelerator check an accept-list". I dared to keep Richard's R-b though.

v2 -> v3:
- check gic version/in-kernel implementation when kvm el2 is set (Peter)

v1 -> v2:
- fixed test ordering: virt && ((kvm && !kvm_el2) || hvf) [Richard]
- tweeked the commit title & message

---
 hw/arm/virt.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)
hw/arm/virt.c +13 -1
index 550a272fbb..6f5339aade 100644
@@ -792,6 +792,13 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
     default:
         g_assert_not_reached();
     }
+
+    if (kvm_enabled() && vms->virt &&
+        (revision != 3 || !kvm_irqchip_in_kernel())) {
+        error_report("KVM EL2 is only supported with in-kernel GICv3");
+        exit(1);
+    }
+
     vms->gic = qdev_new(gictype);
     qdev_prop_set_uint32(vms->gic, "revision", revision);
     qdev_prop_set_uint32(vms->gic, "num-cpu", smp_cpus);
@@ -2066,6 +2073,10 @@ static void virt_post_cpus_gic_realized(VirtMachineState *vms,
             memory_region_init_ram(pvtime, NULL, "pvtime", pvtime_size, NULL);
             memory_region_add_subregion(sysmem, pvtime_reg_base, pvtime);
         }
+        if (!aarch64 && vms->virt) {
+            error_report("vcpu with both EL1_32BIT and HAS_EL2 is not supported");
+            exit(1);
+        }
 
         CPU_FOREACH(cpu) {
             if (pmu) {
@@ -2211,7 +2222,8 @@ static void machvirt_init(MachineState *machine)
         exit(1);
     }
 
-    if (vms->virt && !tcg_enabled() && !qtest_enabled()) {
+    if (vms->virt && !(kvm_enabled() && kvm_arm_el2_supported()) &&
+                     !tcg_enabled() && !qtest_enabled()) {
         error_report("mach-virt: %s does not support providing "
                      "Virtualization extensions to the guest CPU",
                      current_accel_name());
Signature · 1 line
2.49.0
#10 lore raw
In reply to Eric Auger
Quoted 11 lines · Eric Auger

This is candidate for 10.1.

For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.

This series can be found at:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v9
previous:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v8

Applied to target-arm.next, thanks. I'm going to squash in this change

hw/arm/virt.c +7 -2
@@ -2248,8 +2248,13 @@ static void machvirt_init(MachineState *machine)
         exit(1);
     }
 
-    if (vms->virt && !(kvm_enabled() && kvm_arm_el2_supported()) &&
-                     !tcg_enabled() && !qtest_enabled()) {
+    if (vms->virt && kvm_enabled() && !kvm_arm_el2_supported()) {
+        error_report("mach-virt: host kernel KVM does not support providing "
+                     "Virtualization extensions to the guest CPU");
+        exit(1);
+    }
+
+    if (vms->virt && !kvm_enabled() && !tcg_enabled() && !qtest_enabled()) {
         error_report("mach-virt: %s does not support providing "
                      "Virtualization extensions to the guest CPU",
                      current_accel_name());

so we can distinguish "QEMU doesn't support KVM EL2" from "QEMU supports it but the kernel does not".

-- PMM

author
#11 lore raw

Hi Peter,

Quoted 34 lines · Peter Maydell
Quoted 10 lines · Eric Auger

This is candidate for 10.1.

For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.

This series can be found at:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v9
previous:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v8

Applied to target-arm.next, thanks. I'm going to squash in this change

hw/arm/virt.c +7 -2
@@ -2248,8 +2248,13 @@ static void machvirt_init(MachineState *machine)
         exit(1);
     }
 
-    if (vms->virt && !(kvm_enabled() && kvm_arm_el2_supported()) &&
-                     !tcg_enabled() && !qtest_enabled()) {
+    if (vms->virt && kvm_enabled() && !kvm_arm_el2_supported()) {
+        error_report("mach-virt: host kernel KVM does not support providing "
+                     "Virtualization extensions to the guest CPU");
+        exit(1);
+    }
+
+    if (vms->virt && !kvm_enabled() && !tcg_enabled() && !qtest_enabled()) {
         error_report("mach-virt: %s does not support providing "
                      "Virtualization extensions to the guest CPU",
                      current_accel_name());
Quoted 2 lines

so we can distinguish "QEMU doesn't support KVM EL2" from "QEMU supports it but the kernel does not".

Sure.

Thanks!

Eric

Quoted 3 lines

-- PMM