Skip to contentSkip to conversation

hw/arm/virt: Handle HVF in finalize_gic_version()

PATCH Superseded Alexander Graf 3 participants 6 messages Dec 20, 2022 – Dec 21, 2022

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

cover hw/arm/virt: Handle HVF in finalize_gic_version()

The finalize_gic_version() function tries to determine which GIC version the current accelerator / host combination supports. During the initial HVF porting efforts, I didn't realize that I also had to touch this function. Then Zenghui brought up this function as reply to my HVF GICv3 enablement patch - and boy it is a mess.

This patch set cleans up all of the GIC finalization so that we can easily plug HVF in and also hopefully will have a better time extending it in the future. As second step, it explicitly adds HVF support and fails loudly for any unsupported accelerators.

Alex

Alexander Graf (2):
hw/arm/virt: Consolidate GIC finalize logic hw/arm/virt: Make accels in GIC finalize logic explicit

 hw/arm/virt.c         | 199 ++++++++++++++++++++++--------------------
 include/hw/arm/virt.h |  12 +--
 2 files changed, 110 insertions(+), 101 deletions(-)
Signature · 1 line
2.37.1 (Apple Git-137.1)
#1 lore raw

PATCH 1/2 hw/arm/virt: Consolidate GIC finalize logic

Up to now, the finalize_gic_version() code open coded what is essentially a support bitmap match between host/emulation environment and desired target GIC type.

This open coding leads to undesirable side effects. For example, a VM with KVM and -smp 10 will automatically choose GICv3 while the same command line with TCG will stay on GICv2 and fail the launch.

This patch combines the TCG and KVM matching code paths by making everything a 2 pass process. First, we determine which GIC versions the current environment is able to support, then we go through a single state machine to determine which target GIC mode that means for us.

After this patch, the only user noticable changes should be consolidated error messages as well as TCG -M virt supporting -smp > 8 automatically.

---
 hw/arm/virt.c         | 198 ++++++++++++++++++++++--------------------
 include/hw/arm/virt.h |  12 +--
 2 files changed, 108 insertions(+), 102 deletions(-)
hw/arm/virt.c +102 -96
index 04eb6c201d..c79f5b6a66 100644
@@ -1820,6 +1820,84 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits)
     }
 }
 
+static VirtGICType finalize_gic_version_do(const char *accel_name,
+                                           VirtGICType gic_version,
+                                           int gics_supported,
+                                           unsigned int max_cpus)
+{
+    /* Convert host/max/nosel to GIC version number */
+    switch (gic_version) {
+    case VIRT_GIC_VERSION_HOST:
+        if (kvm_enabled()) {
+            /* For KVM, -cpu host means -cpu max */
+            return finalize_gic_version_do(accel_name, VIRT_GIC_VERSION_MAX,
+                                           gics_supported, max_cpus);
+        }
+
+        error_report("gic-version=host requires KVM");
+        exit(1);
+        break;
+    case VIRT_GIC_VERSION_MAX:
+        if (gics_supported & VIRT_GIC_VERSION_4) {
+            gic_version = VIRT_GIC_VERSION_4;
+        } else if (gics_supported & VIRT_GIC_VERSION_3) {
+            gic_version = VIRT_GIC_VERSION_3;
+        } else {
+            gic_version = VIRT_GIC_VERSION_2;
+        }
+        break;
+    case VIRT_GIC_VERSION_NOSEL:
+        if ((gics_supported & VIRT_GIC_VERSION_2) && max_cpus <= GIC_NCPU) {
+            gic_version = VIRT_GIC_VERSION_2;
+        } else if (gics_supported & VIRT_GIC_VERSION_3) {
+            /*
+             * in case the host does not support v2 emulation or
+             * the end-user requested more than 8 VCPUs we now default
+             * to v3. In any case defaulting to v2 would be broken.
+             */
+            gic_version = VIRT_GIC_VERSION_3;
+        } else if (max_cpus > GIC_NCPU) {
+            error_report("%s only supports GICv2 emulation but more than 8 "
+                         "vcpus are requested", accel_name);
+            exit(1);
+        }
+        break;
+    case VIRT_GIC_VERSION_2:
+    case VIRT_GIC_VERSION_3:
+    case VIRT_GIC_VERSION_4:
+        break;
+    }
+
+    /* Check chosen version is effectively supported */
+    switch (gic_version) {
+    case VIRT_GIC_VERSION_2:
+        if (!(gics_supported & VIRT_GIC_VERSION_2)) {
+            error_report("%s does not support GICv2 emulation", accel_name);
+            exit(1);
+        }
+        break;
+    case VIRT_GIC_VERSION_3:
+        if (!(gics_supported & VIRT_GIC_VERSION_3)) {
+            error_report("%s does not support GICv3 emulation", accel_name);
+            exit(1);
+        }
+        break;
+    case VIRT_GIC_VERSION_4:
+        if (!(gics_supported & VIRT_GIC_VERSION_4)) {
+            error_report("%s does not support GICv4 emulation, is virtualization=on?",
+                         accel_name);
+            exit(1);
+        }
+        break;
+    default:
+        error_report("logic error in finalize_gic_version");
+        exit(1);
+        break;
+    }
+
+    return gic_version;
+}
+
 /*
  * finalize_gic_version - Determines the final gic_version
  * according to the gic-version property
@@ -1828,118 +1906,46 @@ static void virt_set_memmap(VirtMachineState *vms, int pa_bits)
  */
 static void finalize_gic_version(VirtMachineState *vms)
 {
+    const char *accel_name = current_accel_name();
     unsigned int max_cpus = MACHINE(vms)->smp.max_cpus;
+    int gics_supported = 0;
 
-    if (kvm_enabled()) {
-        int probe_bitmap;
-
-        if (!kvm_irqchip_in_kernel()) {
-            switch (vms->gic_version) {
-            case VIRT_GIC_VERSION_HOST:
-                warn_report(
-                    "gic-version=host not relevant with kernel-irqchip=off "
-                     "as only userspace GICv2 is supported. Using v2 ...");
-                return;
-            case VIRT_GIC_VERSION_MAX:
-            case VIRT_GIC_VERSION_NOSEL:
-                vms->gic_version = VIRT_GIC_VERSION_2;
-                return;
-            case VIRT_GIC_VERSION_2:
-                return;
-            case VIRT_GIC_VERSION_3:
-                error_report(
-                    "gic-version=3 is not supported with kernel-irqchip=off");
-                exit(1);
-            case VIRT_GIC_VERSION_4:
-                error_report(
-                    "gic-version=4 is not supported with kernel-irqchip=off");
-                exit(1);
-            }
-        }
+    /* Determine which GIC versions the current environment supports */
+    if (kvm_enabled() && kvm_irqchip_in_kernel()) {
+        int probe_bitmap = kvm_arm_vgic_probe();
 
-        probe_bitmap = kvm_arm_vgic_probe();
         if (!probe_bitmap) {
             error_report("Unable to determine GIC version supported by host");
             exit(1);
         }
 
-        switch (vms->gic_version) {
-        case VIRT_GIC_VERSION_HOST:
-        case VIRT_GIC_VERSION_MAX:
-            if (probe_bitmap & KVM_ARM_VGIC_V3) {
-                vms->gic_version = VIRT_GIC_VERSION_3;
-            } else {
-                vms->gic_version = VIRT_GIC_VERSION_2;
-            }
-            return;
-        case VIRT_GIC_VERSION_NOSEL:
-            if ((probe_bitmap & KVM_ARM_VGIC_V2) && max_cpus <= GIC_NCPU) {
-                vms->gic_version = VIRT_GIC_VERSION_2;
-            } else if (probe_bitmap & KVM_ARM_VGIC_V3) {
-                /*
-                 * in case the host does not support v2 in-kernel emulation or
-                 * the end-user requested more than 8 VCPUs we now default
-                 * to v3. In any case defaulting to v2 would be broken.
-                 */
-                vms->gic_version = VIRT_GIC_VERSION_3;
-            } else if (max_cpus > GIC_NCPU) {
-                error_report("host only supports in-kernel GICv2 emulation "
-                             "but more than 8 vcpus are requested");
-                exit(1);
-            }
-            break;
-        case VIRT_GIC_VERSION_2:
-        case VIRT_GIC_VERSION_3:
-            break;
-        case VIRT_GIC_VERSION_4:
-            error_report("gic-version=4 is not supported with KVM");
-            exit(1);
+        if (probe_bitmap & KVM_ARM_VGIC_V2) {
+            gics_supported |= VIRT_GIC_VERSION_2;
         }
-
-        /* Check chosen version is effectively supported by the host */
-        if (vms->gic_version == VIRT_GIC_VERSION_2 &&
-            !(probe_bitmap & KVM_ARM_VGIC_V2)) {
-            error_report("host does not support in-kernel GICv2 emulation");
-            exit(1);
-        } else if (vms->gic_version == VIRT_GIC_VERSION_3 &&
-                   !(probe_bitmap & KVM_ARM_VGIC_V3)) {
-            error_report("host does not support in-kernel GICv3 emulation");
-            exit(1);
+        if (probe_bitmap & KVM_ARM_VGIC_V3) {
+            gics_supported |= VIRT_GIC_VERSION_3;
         }
-        return;
-    }
-
-    /* TCG mode */
-    switch (vms->gic_version) {
-    case VIRT_GIC_VERSION_NOSEL:
-        vms->gic_version = VIRT_GIC_VERSION_2;
-        break;
-    case VIRT_GIC_VERSION_MAX:
+    } else if (kvm_enabled() && !kvm_irqchip_in_kernel()) {
+        /* KVM w/o kernel irqchip can only deal with GICv2 */
+        gics_supported |= VIRT_GIC_VERSION_2;
+        accel_name = "KVM with kernel-irqchip=off";
+    } else {
+        gics_supported |= VIRT_GIC_VERSION_2;
         if (module_object_class_by_name("arm-gicv3")) {
-            /* CONFIG_ARM_GICV3_TCG was set */
+            gics_supported |= VIRT_GIC_VERSION_3;
             if (vms->virt) {
                 /* GICv4 only makes sense if CPU has EL2 */
-                vms->gic_version = VIRT_GIC_VERSION_4;
-            } else {
-                vms->gic_version = VIRT_GIC_VERSION_3;
+                gics_supported |= VIRT_GIC_VERSION_4;
             }
-        } else {
-            vms->gic_version = VIRT_GIC_VERSION_2;
         }
-        break;
-    case VIRT_GIC_VERSION_HOST:
-        error_report("gic-version=host requires KVM");
-        exit(1);
-    case VIRT_GIC_VERSION_4:
-        if (!vms->virt) {
-            error_report("gic-version=4 requires virtualization enabled");
-            exit(1);
-        }
-        break;
-    case VIRT_GIC_VERSION_2:
-    case VIRT_GIC_VERSION_3:
-        break;
     }
+
+    /*
+     * Then convert helpers like host/max to concrete GIC versions and ensure
+     * the desired version is supported
+     */
+    vms->gic_version = finalize_gic_version_do(accel_name, vms->gic_version,
+                                               gics_supported, max_cpus);
 }
 
 /*
include/hw/arm/virt.h +6 -6
index c7dd59d7f1..365d19f7a3 100644
@@ -109,12 +109,12 @@ typedef enum VirtMSIControllerType {
 } VirtMSIControllerType;
 
 typedef enum VirtGICType {
-    VIRT_GIC_VERSION_MAX,
-    VIRT_GIC_VERSION_HOST,
-    VIRT_GIC_VERSION_2,
-    VIRT_GIC_VERSION_3,
-    VIRT_GIC_VERSION_4,
-    VIRT_GIC_VERSION_NOSEL,
+    VIRT_GIC_VERSION_MAX = 0,
+    VIRT_GIC_VERSION_HOST = 1,
+    VIRT_GIC_VERSION_NOSEL = 2,
+    VIRT_GIC_VERSION_2 = (1 << 2),
+    VIRT_GIC_VERSION_3 = (1 << 3),
+    VIRT_GIC_VERSION_4 = (1 << 4),
 } VirtGICType;
 
 struct VirtMachineClass {
Signature · 1 line
2.37.1 (Apple Git-137.1)
#2 lore raw

Re: [PATCH 1/2] hw/arm/virt: Consolidate GIC finalize logic

Quoted 10 lines · Alexander Graf
include/hw/arm/virt.h +0 -2
index c7dd59d7f1..365d19f7a3 100644
@@ -109,12 +109,12 @@ typedef enum VirtMSIControllerType {
 } VirtMSIControllerType;
 
 typedef enum VirtGICType {
-    VIRT_GIC_VERSION_MAX,
-    VIRT_GIC_VERSION_HOST,

include/hw/arm/virt.h typedef enum VirtMSIControllerType { · line 117

Quoted 10 lines
include/hw/arm/virt.h +6 -4
-    VIRT_GIC_VERSION_2,
-    VIRT_GIC_VERSION_3,
-    VIRT_GIC_VERSION_4,
-    VIRT_GIC_VERSION_NOSEL,
+    VIRT_GIC_VERSION_MAX = 0,
+    VIRT_GIC_VERSION_HOST = 1,
+    VIRT_GIC_VERSION_NOSEL = 2,
+    VIRT_GIC_VERSION_2 = (1 << 2),
+    VIRT_GIC_VERSION_3 = (1 << 3),
+    VIRT_GIC_VERSION_4 = (1 << 4),

This would break the ACPI case. When building the MADT, we currently write the raw vms->gic_version value into "GIC version" field of the GICD structure. This happens to work because VIRT_GIC_VERSION_x == x (by luck, I think). We may need to fix build_madt() before this change.

|static void
|build_madt(GArray *table_data, BIOSLinker *linker, VirtMachineState *vms)
|{
|    /* GIC version */
|    build_append_int_noprefix(table_data, vms->gic_version, 1);
#3 lore raw

Re: [PATCH 1/2] hw/arm/virt: Consolidate GIC finalize logic

Hey Zengui,

Quoted 18 lines · Zenghui Yu
Quoted 17 lines · Alexander Graf
include/hw/arm/virt.h +3 -6
index c7dd59d7f1..365d19f7a3 100644
@@ -109,12 +109,12 @@ typedef enum VirtMSIControllerType {
  } VirtMSIControllerType;
 
  typedef enum VirtGICType {
-    VIRT_GIC_VERSION_MAX,
-    VIRT_GIC_VERSION_HOST,
-    VIRT_GIC_VERSION_2,
-    VIRT_GIC_VERSION_3,
-    VIRT_GIC_VERSION_4,
-    VIRT_GIC_VERSION_NOSEL,
+    VIRT_GIC_VERSION_MAX = 0,
+    VIRT_GIC_VERSION_HOST = 1,
+    VIRT_GIC_VERSION_NOSEL = 2,
Quoted 8 lines

include/hw/arm/virt.h typedef enum VirtMSIControllerType { · line 117

Quoted 3 lines
include/hw/arm/virt.h +3 -0
+    VIRT_GIC_VERSION_2 = (1 << 2),
+    VIRT_GIC_VERSION_3 = (1 << 3),
+    VIRT_GIC_VERSION_4 = (1 << 4),

This would break the ACPI case. When building the MADT, we currently write the raw vms->gic_version value into "GIC version" field of the GICD structure. This happens to work because VIRT_GIC_VERSION_x == x (by luck, I think). We may need to fix build_madt() before this change.

Ouch, thanks a lot for the catch! I don't think it's by luck - the versions are put very deliberately at a place where they equal the gic number. But I agree that it's missing a comment - I'll add one for clarification and make sure the defines looks explicit in v2.

Alex

#4 lore raw

PATCH 2/2 hw/arm/virt: Make accels in GIC finalize logic explicit

In reply to Alexander Graf

Let's explicitly list out all accelerators that we support when trying to determine the supported set of GIC versions. KVM was already separate, so the only missing one is HVF which simply reuses all of TCG's emulation code and thus has the same compatibility matrix.

---
 hw/arm/virt.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
hw/arm/virt.c +4 -1
index c79f5b6a66..b7fb473788 100644
@@ -1929,7 +1929,7 @@ static void finalize_gic_version(VirtMachineState *vms)
         /* KVM w/o kernel irqchip can only deal with GICv2 */
         gics_supported |= VIRT_GIC_VERSION_2;
         accel_name = "KVM with kernel-irqchip=off";
-    } else {
+    } else if (tcg_enabled() || hvf_enabled())  {
         gics_supported |= VIRT_GIC_VERSION_2;
         if (module_object_class_by_name("arm-gicv3")) {
             gics_supported |= VIRT_GIC_VERSION_3;
@@ -1938,6 +1938,9 @@ static void finalize_gic_version(VirtMachineState *vms)
                 gics_supported |= VIRT_GIC_VERSION_4;
             }
         }
+    } else {
+        error_report("Unsupported accelerator, can not determine GIC support");
+        exit(1);
     }
 
     /*
Signature · 1 line
2.37.1 (Apple Git-137.1)
#5 lore raw
In reply to Alexander Graf

[ +Eric who wrote finalize_gic_version() ]

Quoted 21 lines · Alexander Graf

The finalize_gic_version() function tries to determine which GIC version the current accelerator / host combination supports. During the initial HVF porting efforts, I didn't realize that I also had to touch this function. Then Zenghui brought up this function as reply to my HVF GICv3 enablement patch - and boy it is a mess.

This patch set cleans up all of the GIC finalization so that we can easily plug HVF in and also hopefully will have a better time extending it in the future. As second step, it explicitly adds HVF support and fails loudly for any unsupported accelerators.

Alex

Alexander Graf (2):
hw/arm/virt: Consolidate GIC finalize logic hw/arm/virt: Make accels in GIC finalize logic explicit

 hw/arm/virt.c         | 199 ++++++++++++++++++++++--------------------
 include/hw/arm/virt.h |  12 +--
 2 files changed, 110 insertions(+), 101 deletions(-)