ARM Nested Virt Support
v5 cover ARM Nested Virt Support
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.
This series can be found at:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v5
Original version from Miguel:
[1] https://lore.kernel.org/all/[email protected]/
version from Haibo:
[2] https://lore.kernel.org/qemu-devel/[email protected]/
History:
v4 -> v5:
- rebase on top of v10.0.0
v3 -> v4:
- fix: only set maint_irq if vms->virt
v2 -> v3:
- KVM EL2 only is set if virtualization option is set
- fixes regression with virtualization=off
- Add checks against unsupported GIC configs until the kernel does
Eric Auger (1):
linux-headers: Update against kvm/next
Haibo Xu (4):
hw/arm: Allow setting KVM vGIC maintenance IRQ target/arm/kvm: Add helper to detect EL2 when using KVM target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported hw/arm/virt: Allow virt extensions with KVM
include/hw/intc/arm_gicv3_common.h | 1 + include/standard-headers/linux/virtio_pci.h | 1 + linux-headers/asm-arm64/kvm.h | 9 +++++---- linux-headers/linux/kvm.h | 3 +++ linux-headers/linux/vhost.h | 4 ++-- target/arm/kvm_arm.h | 7 +++++++ hw/arm/virt.c | 13 ++++++++++++- hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c | 21 +++++++++++++++++++++ target/arm/kvm-stub.c | 5 +++++ target/arm/kvm.c | 21 +++++++++++++++++++++ 11 files changed, 79 insertions(+), 7 deletions(-)
Signature · 1 line
2.49.0
PATCH v5 1/5 linux-headers: Update against kvm/next
This is a linux header update against kvm/next (e9f17038d814)
especially targeted to get nested virt enablement.
--- include/standard-headers/linux/virtio_pci.h | 1 + linux-headers/asm-arm64/kvm.h | 9 +++++---- linux-headers/linux/kvm.h | 3 +++ linux-headers/linux/vhost.h | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-)
include/standard-headers/linux/virtio_pci.h +1 -0
@@ -246,6 +246,7 @@ struct virtio_pci_cfg_cap {#define VIRTIO_ADMIN_CMD_LIST_USE 0x1/* Admin command group type. */ +#define VIRTIO_ADMIN_GROUP_TYPE_SELF 0x0#define VIRTIO_ADMIN_GROUP_TYPE_SRIOV 0x1/* Transitional device admin command. */
linux-headers/asm-arm64/kvm.h +5 -4
@@ -419,10 +419,11 @@ enum {/* Device Control API on vcpu fd */ #define KVM_ARM_VCPU_PMU_V3_CTRL 0 -#define KVM_ARM_VCPU_PMU_V3_IRQ 0 -#define KVM_ARM_VCPU_PMU_V3_INIT 1 -#define KVM_ARM_VCPU_PMU_V3_FILTER 2 -#define KVM_ARM_VCPU_PMU_V3_SET_PMU 3 +#define KVM_ARM_VCPU_PMU_V3_IRQ 0 +#define KVM_ARM_VCPU_PMU_V3_INIT 1 +#define KVM_ARM_VCPU_PMU_V3_FILTER 2 +#define KVM_ARM_VCPU_PMU_V3_SET_PMU 3 +#define KVM_ARM_VCPU_PMU_V3_SET_NR_COUNTERS 4#define KVM_ARM_VCPU_TIMER_CTRL 1#define KVM_ARM_VCPU_TIMER_IRQ_VTIMER 0#define KVM_ARM_VCPU_TIMER_IRQ_PTIMER 1
linux-headers/linux/kvm.h +3 -0
@@ -923,6 +923,9 @@ struct kvm_enable_cap {#define KVM_CAP_X86_APIC_BUS_CYCLES_NS 237#define KVM_CAP_X86_GUEST_MODE 238#define KVM_CAP_ARM_WRITABLE_IMP_ID_REGS 239 +#define KVM_CAP_ARM_EL2 240 +#define KVM_CAP_ARM_EL2_E2H0 241 +#define KVM_CAP_RISCV_MP_STATE_RESET 242struct kvm_irq_routing_irqchip {__u32 irqchip;
linux-headers/linux/vhost.h +2 -2
@@ -28,10 +28,10 @@/* Set current process as the (exclusive) owner of this file descriptor. This * must be called before any other vhost command. Further calls to -* VHOST_OWNER_SET fail until VHOST_OWNER_RESET is called. */ +* VHOST_SET_OWNER fail until VHOST_RESET_OWNER is called. */ #define VHOST_SET_OWNER _IO(VHOST_VIRTIO, 0x01)/* Give up ownership, and reset the device to default values. -* Allows subsequent call to VHOST_OWNER_SET to succeed. */ +* Allows subsequent call to VHOST_SET_OWNER to succeed. */ #define VHOST_RESET_OWNER _IO(VHOST_VIRTIO, 0x02)/* Set up/modify memory layout */
Signature · 1 line
2.49.0
PATCH v5 2/5 hw/arm: Allow setting KVM vGIC maintenance IRQ
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
Allow virt arm machine to set the interrupt ID for the KVM GIC maintenance interrupt.
This setting must be done before the KVM_DEV_ARM_VGIC_CTRL_INIT hence the choice to perform the setting in the GICv3 realize instead of proceeding the same way as kvm_arm_pmu_set_irq().
---
v3 -> v4:
- only set maint_irq if vms->virt
v2 -> v3:
- tweak the commit message and explain why we do not proceed the same way as kvm_arm_pmu_set_irq (Peter)
v1 -> v2:
- [Miguel] replaced the has_virt_extensions by the maintenance irq intid property. [Eric] restored kvm_device_check_attr and kvm_device_access standard usage and conditionally call those if the prop is set.
--- include/hw/intc/arm_gicv3_common.h | 1 + hw/arm/virt.c | 3 +++ hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c | 21 +++++++++++++++++++++ 4 files changed, 26 insertions(+)
include/hw/intc/arm_gicv3_common.h +1 -0
@@ -231,6 +231,7 @@ struct GICv3State {uint32_t num_cpu;uint32_t num_irq;uint32_t revision; +uint32_t maint_irq;bool lpi_enable;bool nmi_support;bool security_extn;
hw/arm/virt.c +3 -0
@@ -831,6 +831,9 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem) OBJECT(mem), &error_fatal);qdev_prop_set_bit (vms->gic,"has-lpi" , true); } + }else if (vms->virt) { +qdev_prop_set_uint32 (vms->gic,"maintenance-interrupt-id" , +ARCH_GIC_MAINT_IRQ ); } }else {if (!kvm_irqchip_in_kernel ()) {
hw/intc/arm_gicv3_common.c +1 -0
@@ -612,6 +612,7 @@ static const Property arm_gicv3_common_properties[] = {DEFINE_PROP_BOOL ("has-lpi" , GICv3State, lpi_enable,0 ),DEFINE_PROP_BOOL ("has-nmi" , GICv3State, nmi_support,0 ),DEFINE_PROP_BOOL ("has-security-extensions" , GICv3State, security_extn,0 ), +DEFINE_PROP_UINT32 ("maintenance-interrupt-id" , GICv3State, maint_irq,0 ),/* * Compatibility property: force 8 bits of physical priority, even * if the CPU being emulated should have fewer.
hw/intc/arm_gicv3_kvm.c +21 -0
@@ -22,6 +22,7 @@#include "qemu/osdep.h" #include "qapi/error.h" #include "hw/intc/arm_gicv3_common.h" +#include "hw/arm/virt.h" #include "qemu/error-report.h" #include "qemu/module.h" #include "system/kvm.h"
@@ -825,6 +826,26 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)return ; } +if (s->maint_irq) { +int ret; + + ret =kvm_device_check_attr (s->dev_fd, +KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ ,0 ); +if (!ret) { +error_setg_errno (errp, errno, +"VGICv3 setting maintenance IRQ is not " +"supported by this host kernel" ); +return ; + } + + ret =kvm_device_access (s->dev_fd,KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ ,0 , + &s->maint_irq, true, errp); +if (ret) { +error_setg_errno (errp, errno,"Failed to set VGIC maintenance IRQ" ); +return ; + } + } + multiple_redist_region_allowed =kvm_device_check_attr (s->dev_fd,KVM_DEV_ARM_VGIC_GRP_ADDR ,KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION );
Signature · 1 line
2.49.0
PATCH v5 3/5 target/arm/kvm: Add helper to detect EL2 when using KVM
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
Introduce query support for KVM_CAP_ARM_EL2.
--- target/arm/kvm_arm.h | 7 +++++++ target/arm/kvm-stub.c | 5 +++++ target/arm/kvm.c | 5 +++++ 3 files changed, 17 insertions(+)
target/arm/kvm_arm.h +7 -0
@@ -190,6 +190,13 @@ bool kvm_arm_sve_supported(void);*/ bool kvm_arm_mte_supported (void ); +/** +* kvm_arm_el2_supported: +* +* Returns true if KVM can enable EL2 and false otherwise. +*/ +bool kvm_arm_el2_supported (void ); +/** * kvm_arm_get_max_vm_ipa_size: * @ms: Machine state handle
target/arm/kvm-stub.c +5 -0
@@ -47,6 +47,11 @@ bool kvm_arm_mte_supported(void)return false; } +bool kvm_arm_el2_supported (void ) +{ +return false; +} +/* * These functions should never actually be called without KVM support. */
target/arm/kvm.c +5 -0
@@ -1769,6 +1769,11 @@ bool kvm_arm_aarch32_supported(void)return kvm_check_extension (kvm_state,KVM_CAP_ARM_EL1_32BIT ); } +bool kvm_arm_el2_supported (void ) +{ +return kvm_check_extension (kvm_state,KVM_CAP_ARM_EL2 ); +} +bool kvm_arm_sve_supported (void ) {return kvm_check_extension (kvm_state,KVM_CAP_ARM_SVE );
Signature · 1 line
2.49.0
PATCH v5 4/5 target/arm: Enable feature ARM_FEATURE_EL2 if EL2 is supported
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
KVM_CAP_ARM_EL2 must be supported by the cpu to enable ARM_FEATURE_EL2. In case the host does support NV, expose the feature.
---
v2 -> v3:
- check pmu->has_el2 on kvm_arch_init_vcpu() when setting KVM_ARM_VCPU_HAS_EL2 feature (Peter)
v1 -> v2:
- remove isar_feature_aa64_aa32_el2 modif in target/arm/cpu.h [Richard] and use el2_supported in kvm_arch_init_vcpu
--- target/arm/kvm.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+)
target/arm/kvm.c +16 -0
@@ -227,6 +227,7 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)*/ int fdarray[3 ];bool sve_supported; +bool el2_supported;bool pmu_supported = false;uint64_t features =0 ;int err;
@@ -246,6 +247,14 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) init.features[0 ] |=1 <<KVM_ARM_VCPU_SVE ; } +/* +* Ask for EL2 if supported. +*/ + el2_supported =kvm_arm_el2_supported (); +if (el2_supported) { + init.features[0 ] |=1 <<KVM_ARM_VCPU_HAS_EL2 ; + } +/* * Ask for Pointer Authentication if supported, so that we get * the unsanitized field values for AA64ISAR1_EL1.
@@ -429,6 +438,10 @@ static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) features |=1ULL <<ARM_FEATURE_AARCH64 ; features |=1ULL <<ARM_FEATURE_GENERIC_TIMER ; +if (el2_supported) { + features |=1ULL <<ARM_FEATURE_EL2 ; + } + ahcf->features = features;return true;
@@ -1889,6 +1902,9 @@ int kvm_arch_init_vcpu(CPUState *cs) cpu->kvm_init_features[0 ] |= (1 <<KVM_ARM_VCPU_PTRAUTH_ADDRESS |1 <<KVM_ARM_VCPU_PTRAUTH_GENERIC ); } +if (cpu->has_el2 &&kvm_arm_el2_supported ()) { + cpu->kvm_init_features[0 ] |=1 <<KVM_ARM_VCPU_HAS_EL2 ; + }/* Do KVM_ARM_VCPU_INIT ioctl */ ret =kvm_arm_vcpu_init (cpu);
Signature · 1 line
2.49.0
PATCH v5 5/5 hw/arm/virt: Allow virt extensions with KVM
In reply to Eric AugerFrom: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
---
v2 -> v3:
- check gic version/in-kernel implementation when kvm el2 is set (Peter)
v1 -> v2:
- fixed test ordering: virt && ((kvm && !kvm_el2) || hvf) [Richard]
- tweeked the commit title & message
--- hw/arm/virt.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-)
hw/arm/virt.c +9 -1
@@ -795,6 +795,13 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)default :g_assert_not_reached (); } + +if (kvm_enabled () && vms->virt && + (revision !=3 || !kvm_irqchip_in_kernel ())) { +error_report ("KVM EL2 only is supported with in-kernel GICv3" ); +exit (1 ); + } + vms->gic =qdev_new (gictype);qdev_prop_set_uint32 (vms->gic,"revision" , revision);qdev_prop_set_uint32 (vms->gic,"num-cpu" , smp_cpus);
@@ -2208,7 +2215,8 @@ static void machvirt_init(MachineState *machine)exit (1 ); } -if (vms->virt && (kvm_enabled () ||hvf_enabled ())) { +if (vms->virt && + ((kvm_enabled () && !kvm_arm_el2_supported ()) ||hvf_enabled ())) {error_report ("mach-virt: %s does not support providing " "Virtualization extensions to the guest CPU" ,current_accel_name ());
Signature · 1 line
2.49.0
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
Quoted 11 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>
Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg -machine virtualization=on, but I don't think that's the ideal behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
Hi Alyssa,
Quoted 11 lines · Alyssa Ross
Quoted 7 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Quoted 8 lines
Quoted 4 lines
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg
I didn’t know '-acell kvm:tcg’ could be used as a fallback mechanism between acceleration modes. May I ask whether do you manage the ‘-cpu’ type for ‘-accel kvm:tcg’ with cpu ‘max’ ?
But more importantly, is this what you’re referring to?
Although,
Quoted 1 line
-machine virtualization=on,
should work for both '-accel kvm’ and ‘-accel tcg’.
Quoted 6 lines
but I don't think that's the ideal
behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.
Quoted 8 lines
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Thanks,
Miguel
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
Quoted 17 lines · Miguel Luis
Hi Alyssa,
Quoted 15 lines · Alyssa Ross
Quoted 11 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>
Quoted 6 lines
Quoted 3 lines
Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg
I didn’t know '-acell kvm:tcg’ could be used as a fallback mechanism between acceleration modes. May I ask whether do you manage the ‘-cpu’ type for ‘-accel kvm:tcg’ with cpu ‘max’ ?
Does it exist?
qemu-system-aarch64: -accel kvm:tcg: invalid accelerator kvm:tcg
Alyssa, didn't you mean -accel kvm or --accel tcg
Quoted 24 lines
But more importantly, is this what you’re referring to?
Although,
Quoted 1 line
-machine virtualization=on,
should work for both '-accel kvm’ and ‘-accel tcg’.
Quoted 6 lines
but I don't think that's the ideal
behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.Quoted 8 lines
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Thanks,
Miguel
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
Hi Eric,
Quoted 22 lines · Eric Auger
Quoted 17 lines · Miguel Luis
Hi Alyssa,
Quoted 15 lines · Alyssa Ross
Quoted 11 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg
I didn’t know '-acell kvm:tcg’ could be used as a fallback mechanism between acceleration modes. May I ask whether do you manage the ‘-cpu’ type for ‘-accel kvm:tcg’ with cpu ‘max’ ?
Does it exist?
qemu-system-aarch64: -accel kvm:tcg: invalid accelerator kvm:tcg
Maybe Alyssa is referring to ‘-M virt,accel=kvm:tcg,virtualization=on,gic-version=3’ ?
The above didn’t triggered any error. Anyhow if the above does what Alyssa’s saying we would just be missing the check for || !tcg_enabled() in this patch, I believe.
Miguel
Quoted 26 lines
Alyssa, didn't you mean -accel kvm or --accel tcg
Quoted 24 lines
But more importantly, is this what you’re referring to?
Although,
Quoted 1 line
-machine virtualization=on,
should work for both '-accel kvm’ and ‘-accel tcg’.
Quoted 6 lines
but I don't think that's the ideal
behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.Quoted 8 lines
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Thanks,
Miguel
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
Hi Miguel,
Quoted 30 lines · Miguel Luis
Hi Eric,
Quoted 28 lines · Eric Auger
Quoted 23 lines · Miguel Luis
Hi Alyssa,
Quoted 11 lines · Alyssa Ross
Quoted 7 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Quoted 7 lines
Quoted 4 lines
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg
I didn’t know '-acell kvm:tcg’ could be used as a fallback mechanism between acceleration modes. May I ask whether do you manage the ‘-cpu’ type for ‘-accel kvm:tcg’ with cpu ‘max’ ?
Quoted 6 lines
Quoted 2 lines
Does it exist?
qemu-system-aarch64: -accel kvm:tcg: invalid accelerator kvm:tcgMaybe Alyssa is referring to ‘-M virt,accel=kvm:tcg,virtualization=on,gic-version=3’ ?
The above didn’t triggered any error. Anyhow if the above does what Alyssa’s saying we would just be missing the check for || !tcg_enabled() in this patch, I believe.
You meant you would report an error only if !tcg_enabled()?
In practice tcg_enabled() does not seem to be set when trying kvm first.
Refering to the ancestor of this feature - which I did not know I acknowledge-,
https://patchwork.ozlabs.org/project/qemu-devel/patch/[email protected]/
it is said:
kvm|tcg tries kvm and fallbacks on tcg if kvm support isn't available
It does not state that each option is evaluated with kvm_enabled() and in case it fails enabling one because the host does not support it, it falls back to TCG.
I checked for instance with already integrated MTE option and if the host does not support it, it does not offer the capability to fall back to TCG.
Adding Peter in TO to get his feeling on this.
Thanks
Eric
Besides
Quoted 28 lines
Miguel
Quoted 24 lines
Alyssa, didn't you mean -accel kvm or --accel tcg
Quoted 23 lines
But more importantly, is this what you’re referring to?
Although,
Quoted 1 line
-machine virtualization=on,
should work for both '-accel kvm’ and ‘-accel tcg’.
Quoted 6 lines
but I don't think that's the ideal
behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.Quoted 8 lines
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Thanks,
Miguel
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
In reply to Miguel LuisHi,
Quoted 30 lines · Miguel Luis
Hi Eric,
Quoted 28 lines · Eric Auger
Quoted 23 lines · Miguel Luis
Hi Alyssa,
Quoted 11 lines · Alyssa Ross
Quoted 7 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Quoted 7 lines
Quoted 4 lines
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg
I didn’t know '-acell kvm:tcg’ could be used as a fallback mechanism between acceleration modes. May I ask whether do you manage the ‘-cpu’ type for ‘-accel kvm:tcg’ with cpu ‘max’ ?
Quoted 6 lines
Quoted 2 lines
Does it exist?
qemu-system-aarch64: -accel kvm:tcg: invalid accelerator kvm:tcgMaybe Alyssa is referring to ‘-M virt,accel=kvm:tcg,virtualization=on,gic-version=3’ ?
The above didn’t triggered any error. Anyhow if the above does what Alyssa’s saying we would just be missing the check for || !tcg_enabled() in this patch, I believe.
After discussion with Paolo, the lack of the CAP should be detected earlier in kvm_init/kvm_arch_init to allow the fallback to TCG. in target/arm/kvm.c kvm_arch_init() some generic caps are checked but none of them are related to machine settings and this code is virt arm machine agnostic.
I checked and adding
if (object_dynamic_cast(OBJECT(ms), TYPE_VIRT_MACHINE)) { VirtMachineState *vms = VIRT_MACHINE(ms);
if (vms->virt && !kvm_arm_el2_supported()) { error_report("KVM does not support nested virtualization"); ret = -EINVAL;
}
}
at the end of the function would do the job. But as I said previously this is not done for other virt arm machine options that are accel specific or require special KVM caps (secure, mte for instance) so it would be a change in the approach.
As pointed out before fallback spirit was rather: "KVM isn't enabled" than for "KVM doesn't support a specific feature".
Thanks
Eric
it's purely the first accelerator that says it can work 1:03
<https://redhat-internal.slack.com/archives/C04KFKV2SE9/p1750331005242709>
pbonzini
Where "can work" is only based on the host.
Quoted 28 lines
Miguel
Quoted 24 lines
Alyssa, didn't you mean -accel kvm or --accel tcg
Quoted 23 lines
But more importantly, is this what you’re referring to?
Although,
Quoted 1 line
-machine virtualization=on,
should work for both '-accel kvm’ and ‘-accel tcg’.
Quoted 6 lines
but I don't think that's the ideal
behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.Quoted 8 lines
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Thanks,
Miguel
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
Quoted 57 lines · Eric Auger
Hi,
Quoted 30 lines · Miguel Luis
Hi Eric,
Quoted 28 lines · Eric Auger
Quoted 23 lines · Miguel Luis
Hi Alyssa,
Quoted 11 lines · Alyssa Ross
Quoted 7 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Quoted 7 lines
Quoted 4 lines
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg
I didn’t know '-acell kvm:tcg’ could be used as a fallback mechanism between acceleration modes. May I ask whether do you manage the ‘-cpu’ type for ‘-accel kvm:tcg’ with cpu ‘max’ ?
Quoted 6 lines
Quoted 2 lines
Does it exist?
qemu-system-aarch64: -accel kvm:tcg: invalid accelerator kvm:tcgMaybe Alyssa is referring to ‘-M virt,accel=kvm:tcg,virtualization=on,gic-version=3’ ?
The above didn’t triggered any error. Anyhow if the above does what Alyssa’s saying we would just be missing the check for || !tcg_enabled() in this patch, I believe.
After discussion with Paolo, the lack of the CAP should be detected earlier in kvm_init/kvm_arch_init to allow the fallback to TCG. in
target/arm/kvm.ckvm_arch_init()some generic caps are checked but none of them are related to machine settings and this code is virt arm machine agnostic.I checked and adding
if (
object_dynamic_cast(OBJECT(ms), TYPE_VIRT_MACHINE)) { VirtMachineState *vms =VIRT_MACHINE(ms);if (
vms->virt&& !kvm_arm_el2_supported()) {error_report("KVM does not support nested virtualization"); ret =-EINVAL;
}
}
Quoted 4 lines
at the end of the function would do the job. But as I said previously this is not done for other virt arm machine options that are accel specific or require special KVM caps (secure, mte for instance) so it would be a change in the approach.
Right — if it's not done for other options it doesn't need to be done here either; I wasn't aware of that. It would be nice if QEMU worked differently, but that's certainly not in scope of this series.
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
In reply to Eric AugerHi Eric,
Quoted 59 lines · Eric Auger
Hi,
Quoted 30 lines · Miguel Luis
Hi Eric,
Quoted 28 lines · Eric Auger
Quoted 23 lines · Miguel Luis
Hi Alyssa,
Quoted 11 lines · Alyssa Ross
Quoted 7 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Quoted 7 lines
Quoted 4 lines
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg
I didn’t know '-acell kvm:tcg’ could be used as a fallback mechanism between acceleration modes. May I ask whether do you manage the ‘-cpu’ type for ‘-accel kvm:tcg’ with cpu ‘max’ ?
Quoted 6 lines
Quoted 2 lines
Does it exist?
qemu-system-aarch64: -accel kvm:tcg: invalid accelerator kvm:tcgMaybe Alyssa is referring to ‘-M virt,accel=kvm:tcg,virtualization=on,gic-version=3’ ?
The above didn’t triggered any error. Anyhow if the above does what Alyssa’s saying we would just be missing the check for || !tcg_enabled() in this patch, I believe.
After discussion with Paolo, the lack of the CAP should be detected earlier in kvm_init/kvm_arch_init to allow the fallback to TCG. in
target/arm/kvm.ckvm_arch_init()some generic caps are checked but none of them are related to machine settings and this code is virt arm machine agnostic.I checked and adding
if (object_dynamic_cast(OBJECT(ms), TYPE_VIRT_MACHINE)) { VirtMachineState *vms = VIRT_MACHINE(ms); if (vms->virt && !kvm_arm_el2_supported()) { error_report("KVM does not support nested virtualization"); ret = -EINVAL; } }
Quoted 5 lines
at the end of the function would do the job. But as I said previously this is not done for other virt arm machine options that are accel specific or require special KVM caps (secure, mte for instance) so it would be a change in the approach.
Got it.
Quoted 3 lines
As pointed out before fallback spirit was rather: "KVM isn't enabled" than for "KVM doesn't support a specific feature".
Thank you for getting to the bottom of it.
Miguel
Quoted 41 lines
Thanks
Eric
it's purely the first accelerator that says it can work 1:03
<https://redhat-internal.slack.com/archives/C04KFKV2SE9/p1750331005242709>
pbonzini
Where "can work" is only based on the host.Quoted 27 lines
Miguel
Quoted 24 lines
Alyssa, didn't you mean -accel kvm or --accel tcg
Quoted 23 lines
But more importantly, is this what you’re referring to?
Although,
Quoted 1 line
-machine virtualization=on,
should work for both '-accel kvm’ and ‘-accel tcg’.
Quoted 6 lines
but I don't think that's the ideal
behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.Quoted 8 lines
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Thanks,
Miguel
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
In reply to Alyssa RossHi Alyssa,
Quoted 13 lines · Alyssa Ross
Quoted 11 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>
Quoted 7 lines
Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg -machine virtualization=on, but I don't think that's the ideal behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.
Hum I think I misundertood your point.
I guess you meant you get "qemu-system-aarch64: mach-virt: KVM does not support providing Virtualization extensions to the guest CPU" which sounds the usual behavior of qemu in case we attempt to run a feature on a host that does not support it.
Quoted 9 lines
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Switching to another accel mode is a policy that is not implemented by qemu. There is no such situation where qemu would switch from kvm accel mode to TCG. You need to implement this policy in your test env I think
Thanks
Eric
Re: [PATCH v5 5/5] hw/arm/virt: Allow virt extensions with KVM
In reply to Alyssa RossHi Alyssa,
Quoted 9 lines · Alyssa Ross
Quoted 7 lines · Eric Auger
From: Haibo Xu <[email protected]>
Up to now virt support on guest has been only supported with TCG. Now it becomes feasible to use it with KVM acceleration.
Also check only in-kernel GICv3 is used along with KVM EL2.
Quoted 7 lines
Quoted 4 lines
Signed-off-by Haibo Xu <[email protected]>Signed-off-by Miguel Luis <[email protected]>Signed-off-by Eric Auger <[email protected]>Reviewed-by Richard Henderson <[email protected]>Hi! From what I can tell, this will produce an error on hosts that don't support nested virtualization when QEMU is invoked with -accel kvm:tcg -machine virtualization=on, but I don't think that's the ideal
Thanks for reporting this issue. This is not an expected behavior. I will respin hopefully this week so I will have give it a try.
What kind of error msg do you get?
Thanks
Eric
Quoted 13 lines
behaviour. It would make more sense for it to fall back to the first permitted accel option that does support running the machine as configured, so if hardware nested virtualization is not supported, it should fall back to TCG.
I maintain an OS development environment that includes scripts for running images in QEMU, where running KVM on those images is a requirement. Currently, those scripts simply force TCG on aarch64. With this change, to take advantage of KVM NV support, I'd have to try to identify in the script whether NV would be supported. QEMU would be in a much better position to determine this and fall back to TCG if it's unsupported, like how the -accel option with multiple values usually works.
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.
Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Hi Marc,
Quoted 10 lines · Marc Zyngier
Hi Eric,
Quoted 6 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.
Quoted 6 lines
Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?
yes I can definitively extend the series to support KVM_ARM_VCPU_HAS_EL2_E2H0.
Eric
Quoted 6 lines
Thanks,
M.
Hi Marc,
Quoted 14 lines · Marc Zyngier
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.
Quoted 7 lines
Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?
Something that pops up is early_kvm_mode_cfg trying to handle nested mode while KVM_ARM_VCPU_HAS_EL2_E2H0 is set.
Miguel
Quoted 7 lines
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 19 lines · Miguel Luis
Hi Marc,
Quoted 16 lines · Marc Zyngier
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.Thanks for respinning this series.
Quoted 8 lines
Quoted 5 lines
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?Something that pops up is
early_kvm_mode_cfgtrying to handle nested mode whileKVM_ARM_VCPU_HAS_EL2_E2H0is set.
Care to elaborate?
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 31 lines · Marc Zyngier
Quoted 25 lines · Miguel Luis
Hi Marc,
Quoted 21 lines · Marc Zyngier
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?
Quoted 5 lines
Quoted 2 lines
Something that pops up is
early_kvm_mode_cfgtrying to handle nested mode whileKVM_ARM_VCPU_HAS_EL2_E2H0is set.Care to elaborate?
Say host is booted in nested mode (kvm-arm.mode=nested) and host's KVM supports both KVM_CAP_ARM_EL2 and KVM_CAP_ARM_E2H0.
A L1 guest boots setting both KVM_ARM_VCPU_HAS_EL2 and KVM_ARM_VCPU_HAS_EL2_E2H0 and guest kernel's command line state kvm-arm.mode=nested.
This splats the kernel from early_kvm_mode_cfg along a malformed early option message.
This is not related to this qemu integration though, should you take it somewhere else, please let me know?
Thanks
Miguel
Quoted 4 lines
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 41 lines · Miguel Luis
Quoted 32 lines · Marc Zyngier
Quoted 25 lines · Miguel Luis
Hi Marc,
Quoted 21 lines · Marc Zyngier
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?Quoted 5 lines
Quoted 2 lines
Something that pops up is
early_kvm_mode_cfgtrying to handle nested mode whileKVM_ARM_VCPU_HAS_EL2_E2H0is set.Care to elaborate?
Say host is booted in nested mode (kvm-arm.mode=nested) and host's KVM supports both
KVM_CAP_ARM_EL2andKVM_CAP_ARM_E2H0.
Quoted 6 lines
A L1 guest boots setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0and guest kernel's command line state kvm-arm.mode=nested.This splats the kernel from
early_kvm_mode_cfgalong a malformed early option message.
BEBKAC. You are asking for nested on a (virtual) machine that doesn't support it, and the kernel tells you so with a warning. Try the same thing on a physical machine that doesn't have NV, and observe the result.
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Hi Marc,
Quoted 49 lines · Marc Zyngier
Quoted 45 lines · Miguel Luis
Quoted 32 lines · Marc Zyngier
Quoted 25 lines · Miguel Luis
Hi Marc,
Quoted 21 lines · Marc Zyngier
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?Quoted 5 lines
Quoted 2 lines
Something that pops up is
early_kvm_mode_cfgtrying to handle nested mode whileKVM_ARM_VCPU_HAS_EL2_E2H0is set.Care to elaborate?
Say host is booted in nested mode (kvm-arm.mode=nested) and host's KVM supports both
KVM_CAP_ARM_EL2andKVM_CAP_ARM_E2H0.A L1 guest boots setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0and guest kernel's command line state kvm-arm.mode=nested.
Quoted 8 lines
Quoted 2 lines
This splats the kernel from
early_kvm_mode_cfgalong a malformed early option message.BEBKAC. You are asking for nested on a (virtual) machine that doesn't support it, and the kernel tells you so with a warning. Try the same thing on a physical machine that doesn't have NV, and observe the result.
Ack.
I find trying them a great way to improve resilience.
I’ve tried the scenarios below which have similar results on the guest:
1.
Host: kvm-arm.mode=nested
L1 Guest: kvm-arm.mode=nvhe setting both KVM_ARM_VCPU_HAS_EL2 and KVM_ARM_VCPU_HAS_EL2_E2H0
Result on the guest: No early_kvm_mode_cfg splat, boot proceeds, ends up in a hard lockup splat.
2.
Host: kvm-arm.mode=nested
L1 Guest: kvm-arm.mode=nested setting both KVM_ARM_VCPU_HAS_EL2 and KVM_ARM_VCPU_HAS_EL2_E2H0
Result on the guest: Splat at early_kvm_mode_cfg, boot proceeds, ends up in hard lockup splat.
Does this means there’s a default fallback mode in which nv gets on when kvm-arm.mode fed to the guest kernel cmdline differs from the expected?
Miguel
Quoted 4 lines
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 66 lines · Miguel Luis
Hi Marc,
Quoted 49 lines · Marc Zyngier
Quoted 45 lines · Miguel Luis
Quoted 32 lines · Marc Zyngier
Quoted 25 lines · Miguel Luis
Hi Marc,
Quoted 21 lines · Marc Zyngier
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?Quoted 5 lines
Quoted 2 lines
Something that pops up is
early_kvm_mode_cfgtrying to handle nested mode whileKVM_ARM_VCPU_HAS_EL2_E2H0is set.Care to elaborate?
Say host is booted in nested mode (kvm-arm.mode=nested) and host's KVM supports both
KVM_CAP_ARM_EL2andKVM_CAP_ARM_E2H0.A L1 guest boots setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0and guest kernel's command line state kvm-arm.mode=nested.Quoted 8 lines
Quoted 2 lines
This splats the kernel from
early_kvm_mode_cfgalong a malformed early option message.BEBKAC. You are asking for nested on a (virtual) machine that doesn't support it, and the kernel tells you so with a warning. Try the same thing on a physical machine that doesn't have NV, and observe the result.
Ack.
I find trying them a great way to improve resilience.
I’ve tried the scenarios below which have similar results on the guest:
Quoted 7 lines
1.
Host: kvm-arm.mode=nestedL1 Guest: kvm-arm.mode=nvhe setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0Result on the guest: No
early_kvm_mode_cfgsplat, boot proceeds, ends up in a hard lockup splat.
Setting kvm-arm.mode=nvhe when KVM_ARM_VCPU_HAS_EL2_E2H0 is set is a tautology. The very definition of nVHE is that HCR_EL2.E2H=0.
Quoted 8 lines
2.
Host: kvm-arm.mode=nestedL1 Guest: kvm-arm.mode=nested setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0Result on the guest: Splat at
early_kvm_mode_cfg, boot proceeds, ends up in hard lockup splat.
I don't see any of these lockups with kvmtool. See this:
for an example of a boot with both capabilities set and the nonsense "nested" on the command-line (your #2).
Quoted 2 lines
Does this means there’s a default fallback mode in which nv gets on when kvm-arm.mode fed to the guest kernel cmdline differs from the expected?
I don't understand your question. We have two modes of operation:
- HAS_EL2 enables NV on the host, and additionally enables recursive NV. As a consequence, HCR_EL2.E2H is RES1. This is how NV will be supported long term.
- HAS_EL2_E2H0 restricts the above by not exposing NV to the guest, and enforcing HCR_EL2.E2H to be RES0. I expect this to gradually be removed from implementations, and eventually disappear.
As you can see, there is no "fallback mode". You pick the mode you want based on the guest you want to run and the capabilities of the hardware.
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 87 lines · Marc Zyngier
Quoted 66 lines · Miguel Luis
Hi Marc,
Quoted 49 lines · Marc Zyngier
Quoted 45 lines · Miguel Luis
Quoted 32 lines · Marc Zyngier
Quoted 25 lines · Miguel Luis
Hi Marc,
Quoted 21 lines · Marc Zyngier
Hi Eric,
Quoted 7 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?Quoted 5 lines
Quoted 2 lines
Something that pops up is
early_kvm_mode_cfgtrying to handle nested mode whileKVM_ARM_VCPU_HAS_EL2_E2H0is set.Care to elaborate?
Say host is booted in nested mode (kvm-arm.mode=nested) and host's KVM supports both
KVM_CAP_ARM_EL2andKVM_CAP_ARM_E2H0.A L1 guest boots setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0and guest kernel's command line state kvm-arm.mode=nested.Quoted 8 lines
Quoted 2 lines
This splats the kernel from
early_kvm_mode_cfgalong a malformed early option message.BEBKAC. You are asking for nested on a (virtual) machine that doesn't support it, and the kernel tells you so with a warning. Try the same thing on a physical machine that doesn't have NV, and observe the result.
Ack.
I find trying them a great way to improve resilience.
I’ve tried the scenarios below which have similar results on the guest:Quoted 7 lines
1.
Host: kvm-arm.mode=nestedL1 Guest: kvm-arm.mode=nvhe setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0Result on the guest: No
early_kvm_mode_cfgsplat, boot proceeds, ends up in a hard lockup splat.Setting kvm-arm.mode=nvhe when
KVM_ARM_VCPU_HAS_EL2_E2H0is set is a tautology. The very definition of nVHE is that HCR_EL2.E2H=0.Quoted 4 lines
2.
Host: kvm-arm.mode=nested
Quoted 8 lines
Quoted 4 lines
L1 Guest: kvm-arm.mode=nested setting both
KVM_ARM_VCPU_HAS_EL2andKVM_ARM_VCPU_HAS_EL2_E2H0Result on the guest: Splat at
early_kvm_mode_cfg, boot proceeds, ends up in hard lockup splat.I don't see any of these lockups with kvmtool. See this:
Could you try bigger values for -c and check whether you can reproduce the issue?
Quoted 13 lines
for an example of a boot with both capabilities set and the nonsense "nested" on the command-line (your #2).
Quoted 2 lines
Does this means there’s a default fallback mode in which nv gets on when kvm-arm.mode fed to the guest kernel cmdline differs from the expected?
I don't understand your question. We have two modes of operation:
-
HAS_EL2enables NV on the host, and additionally enables recursive NV. As a consequence, HCR_EL2.E2H is RES1. This is how NV will be supported long term.
Quoted 8 lines
-
HAS_EL2_E2H0restricts the above by not exposing NV to the guest, and enforcing HCR_EL2.E2H to be RES0. I expect this to gradually be removed from implementations, and eventually disappear.As you can see, there is no "fallback mode". You pick the mode you want based on the guest you want to run and the capabilities of the hardware.
I’m now suspecting the lockups might have a different reason than guest’s mode.
Thanks
Miguel
Quoted 4 lines
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 8 lines · Miguel Luis
Quoted 8 lines · Marc Zyngier
Quoted 1 line · Miguel Luis
Quoted 7 lines
Quoted 5 lines
Result on the guest: Splat at
early_kvm_mode_cfg, boot proceeds, ends up in hard lockup splat.I don't see any of these lockups with kvmtool. See this:
Could you try bigger values for -c and check whether you can reproduce the issue?
How about 512, which is the maximum we support, and works for me with an oversubscribing ratio of 42:1? More importantly, how about *you* analyse the issue and report something that is actually actionable?
Quoted 1 line
I’m now suspecting the lockups might have a different reason than guest’s mode.
I'm shocked! :)
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Hi Marc, Miguel,
Quoted 10 lines · Marc Zyngier
Hi Eric,
Quoted 6 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.
Quoted 6 lines
Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?
While respinning I'd rather not support KVM_ARM_VCPU_HAS_EL2_E2H0 in next respin. The code is mostly reviewed and I would rather add support for this in a separate series. It will most probably rely on a change of the virtualization property type to support a "restricted" enum value.
Thanks
Eric
Quoted 6 lines
Thanks,
M.
________________________________
De: Eric Auger <[email protected]>
Enviado: quinta-feira, junho 19, 2025 8:19:47 da manhã
Para: Marc Zyngier <[email protected]>
Cc: [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; Miguel Luis <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>; [email protected] <[email protected]>
Assunto: Re: [PATCH v5 0/5] ARM Nested Virt Support
Hi Marc, Miguel,
Quoted 10 lines · Marc Zyngier
Hi Eric,
Quoted 6 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.
Quoted 6 lines
Thanks for respinning this series.
Do you have any plan to support the non-VHE version of the NV support (as advertised by
KVM_CAP_ARM_EL2_E2H0)? It would allow running lesser hypervisors (such as *cough* Xen *cough*), which completely rely on HCR_EL2.E2H being 0?
While respinning I'd rather not support KVM_ARM_VCPU_HAS_EL2_E2H0 in next respin. The code is mostly reviewed and I would rather add support for this in a separate series. It will most probably rely on a change of the virtualization property type to support a "restricted" enum value.
Ack!
Thank you.
Miguel
Thanks
Eric
Quoted 6 lines
Thanks,
M.
Hi Eric,
Quoted 8 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.
For the current kvmarm/next the guest also needs “kvm-arm.mode=nested” I believe.
Quoted 4 lines
This series can be found at:
https://github.com/eauger/qemu/tree/v10.0.0-nv-v5
Thanks for the respin.
Cheers
Miguel
Quoted 42 lines
Original version from Miguel:
[1] https://lore.kernel.org/all/[email protected]/
version from Haibo:
[2] https://lore.kernel.org/qemu-devel/[email protected]/History:
v4 -> v5:
- rebase on top of v10.0.0v3 -> v4:
- fix: only setmaint_irqifvms->virtv2 -> v3:
- KVM EL2 only is set if virtualization option is set
- fixes regression with virtualization=off
- Add checks against unsupported GIC configs until the kernel doesEric Auger (1):
linux-headers: Update against kvm/nextHaibo Xu (4):
hw/arm: Allow setting KVM vGIC maintenance IRQ target/arm/kvm: Add helper to detect EL2 when using KVM target/arm: Enable featureARM_FEATURE_EL2if EL2 is supported hw/arm/virt: Allow virt extensions with KVMinclude/hw/intc/arm_gicv3_common.h | 1 + include/standard-headers/linux/virtio_pci.h | 1 + linux-headers/asm-arm64/kvm.h | 9 +++++---- linux-headers/linux/kvm.h | 3 +++ linux-headers/linux/vhost.h | 4 ++-- target/arm/kvm_arm.h | 7 +++++++ hw/arm/virt.c | 13 ++++++++++++- hw/intc/arm_gicv3_common.c | 1 + hw/intc/arm_gicv3_kvm.c | 21 +++++++++++++++++++++ target/arm/kvm-stub.c | 5 +++++ target/arm/kvm.c | 21 +++++++++++++++++++++ 11 files changed, 79 insertions(+), 7 deletions(-)Signature · 1 line
2.49.0
Quoted 8 lines · Miguel Luis
Hi Eric,
Quoted 5 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
Quoted 6 lines
Quoted 3 lines
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.For the current kvmarm/next the guest also needs
“kvm-arm.mode=nested” I believe.
No, unless you want the guest itself to be NV-capable.
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 14 lines · Marc Zyngier
Quoted 8 lines · Miguel Luis
Hi Eric,
Quoted 5 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
Quoted 9 lines
Quoted 6 lines
Quoted 3 lines
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.For the current kvmarm/next the guest also needs
“kvm-arm.mode=nested” I believe.No, unless you want the guest itself to be NV-capable.
Correct, I got carried away with some mode combinations. Maybe we should depict here more broadly how NV might be used with different mode combinations. I'll think about this further ahead.
As far this series go I couldn't found any issue booting a L1 guest with virtualization=on and a L2 guest with virtualization=off.
Miguel
Quoted 4 lines
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Hi Miguel, Marc,
Quoted 12 lines · Miguel Luis
Quoted 12 lines · Marc Zyngier
Quoted 7 lines · Miguel Luis
Hi Eric,
Quoted 5 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 3 lines
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.For the current kvmarm/next the guest also needs
“kvm-arm.mode=nested” I believe.No, unless you want the guest itself to be NV-capable.
Effectively you don't need "kvm-arm.mode=nested" except if you want want multiple level of nesting.
Quoted 7 lines
Quoted 1 line
Correct, I got carried away with some mode combinations. Maybe we should depict here more broadly how NV might be used with different mode combinations. I'll think about this further ahead.
As far this series go I couldn't found any issue booting a L1 guest with virtualization=on and a L2 guest with virtualization=off.
on my end I tested with various untouched L2 guests (debian, fed, rhel) in 4kB/4kB/4KB page size mode (host, L1, L2). Those configs were successful.
with 64kB/64kB/64kB configs I am less lucky atm. One one machine I cannot boot L1 with virtualization=on. On the other I can boot L1 but cannot boot L2.
Trying my best to debug a little bit further with my setup. Anyway if somebody else can try 64kB configs, it would help to confirm whether there are pending issues. I don't think they are related to this qemu integration series though.
Thanks
Eric
Quoted 8 lines
Miguel
Quoted 4 lines
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.
Quoted 16 lines · Eric Auger
Hi Miguel, Marc,
Quoted 12 lines · Miguel Luis
Quoted 12 lines · Marc Zyngier
Quoted 7 lines · Miguel Luis
Hi Eric,
Quoted 5 lines · Eric Auger
Now that ARM nested virt has landed in kvm/next, let's turn the series into a PATCH series. The linux header update was made against kvm/next.
Quoted 8 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 3 lines
For gaining virt functionality in KVM accelerated L1, The host needs to be booted with "kvm-arm.mode=nested" option and qemu needs to be invoked
with: -machine virt,virtualization=on.For the current kvmarm/next the guest also needs
“kvm-arm.mode=nested” I believe.No, unless you want the guest itself to be NV-capable.
Effectively you don't need "kvm-arm.mode=nested" except if you want want multiple level of nesting.
You need kvm-arm.mode=nested on the command-line of a kernel that acts as a host exposing KVM_CAP_ARM_EL2. This is obviously a recursive property, but that's for the user to decide if they want it or not.
Quoted 9 lines
Quoted 6 lines
Correct, I got carried away with some mode combinations. Maybe we should depict here more broadly how NV might be used with different mode combinations. I'll think about this further ahead.
As far this series go I couldn't found any issue booting a L1 guest with virtualization=on and a L2 guest with virtualization=off.
on my end I tested with various untouched L2 guests (debian, fed, rhel) in 4kB/4kB/4KB page size mode (host, L1, L2). Those configs were successful.
I'm doing 4/4/4 (x1e) and 16/16/4 (m2). I'm also doing 4/4/4/4/4, but that's not very reliable yet,
Quoted 3 lines
with 64kB/64kB/64kB configs I am less lucky atm. One one machine I cannot boot L1 with virtualization=on. On the other I can boot L1 but cannot boot L2.
I think x1e supports 64k pages, but I'd need to try. Not exactly the highest priority on my list of things to debug.
Quoted 4 lines
Trying my best to debug a little bit further with my setup. Anyway if somebody else can try 64kB configs, it would help to confirm whether there are pending issues. I don't think they are related to this qemu integration series though.
Yeah, definitely a kernel issue IMO.
Thanks,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.