spapr: add KVM support to the XIVE interrupt mode
cover spapr: add KVM support to the XIVE interrupt mode
Hello,
Following the 'dual' IRQ backend, this series adds KVM support to the XIVE interrupt mode.
The first patches introduce the XIVE KVM device, state synchronization and migration support under KVM. The second part of the patchset modifies the XICS and XIVE interrupt models to add KVM support to the 'dual' IRQ backend.
This is a first round to check that the interfaces with Linux/KVM are well in place.
GitHub trees available here :
QEMU sPAPR:
https://github.com/legoater/qemu/commits/xive-next
Linux/KVM:
https://github.com/legoater/linux/commits/xive-5.0
OPAL:
https://github.com/legoater/skiboot/commits/xive
Thanks,
C.
Cédric Le Goater (13):
linux-headers: update to 5.0
spapr/xive: add KVM support
spapr/xive: add state synchronization with KVM spapr/xive: introduce a VM state change handler spapr/xive: add migration support for KVM
spapr/xive: fix migration of the XiveTCTX under TCG ppc/xics: introduce a icp_kvm_connect() routine spapr/rtas: modify spapr_rtas_register() to remove RTAS handlers
sysbus: add a sysbus_mmio_unmap() helper
spapr: introduce routines to delete the KVM IRQ device
spapr: check for the activation of the KVM IRQ device spapr/xics: ignore the lower 4K in the IRQ number space
spapr: add KVM support to the 'dual' machine
default-configs/ppc64-softmmu.mak | 1 + include/hw/ppc/spapr.h | 4 + include/hw/ppc/spapr_xive.h | 26 + include/hw/ppc/xics.h | 1 + include/hw/ppc/xive.h | 24 + include/hw/sysbus.h | 1 + linux-headers/asm-powerpc/kvm.h | 46 ++ linux-headers/linux/kvm.h | 9 + target/ppc/kvm_ppc.h | 6 + hw/core/sysbus.c | 10 + hw/intc/spapr_xive.c | 83 ++- hw/intc/spapr_xive_kvm.c | 886 ++++++++++++++++++++++++++++++ hw/intc/xics_kvm.c | 154 +++++- hw/intc/xive.c | 45 +- hw/ppc/spapr_irq.c | 114 +++- hw/ppc/spapr_rtas.c | 2 +- target/ppc/kvm.c | 7 + hw/intc/Makefile.objs | 1 + 18 files changed, 1363 insertions(+), 57 deletions(-) create mode 100644 hw/intc/spapr_xive_kvm.c
Signature · 1 line
2.20.1
PATCH 1/13 linux-headers: update to 5.0
These changes provide the interface with the KVM device implementing the XIVE native exploitation interrupt mode. Also used to retrieve the state of the KVM device for the monitor usage and for migration.
Available from :
https://github.com/legoater/linux/commits/xive-5.0
--- linux-headers/asm-powerpc/kvm.h | 46 +++++++++++++++++++++++++++++++++ linux-headers/linux/kvm.h | 9 +++++++ 2 files changed, 55 insertions(+)
linux-headers/asm-powerpc/kvm.h +46 -0
@@ -480,6 +480,8 @@ struct kvm_ppc_cpu_char {#define KVM_REG_PPC_ICP_PPRI_SHIFT 16/* pending irq priority */ #define KVM_REG_PPC_ICP_PPRI_MASK 0xff +#define KVM_REG_PPC_NVT_STATE (KVM_REG_PPC | KVM_REG_SIZE_U256 | 0x8d) +/* Device control API: PPC-specific devices */ #define KVM_DEV_MPIC_GRP_MISC 1#define KVM_DEV_MPIC_BASE_ADDR 0/* 64-bit */
@@ -675,4 +677,48 @@ struct kvm_ppc_cpu_char {#define KVM_XICS_PRESENTED (1ULL << 43)#define KVM_XICS_QUEUED (1ULL << 44) +/* POWER9 XIVE Native Interrupt Controller */ +#define KVM_DEV_XIVE_GRP_CTRL 1 +#define KVM_DEV_XIVE_GET_ESB_FD 1 +#define KVM_DEV_XIVE_GET_TIMA_FD 2 +#define KVM_DEV_XIVE_VC_BASE 3 +#define KVM_DEV_XIVE_SAVE_EQ_PAGES 4 +#define KVM_DEV_XIVE_GRP_SOURCES 2/* 64-bit source attributes */ +#define KVM_DEV_XIVE_GRP_SYNC 3/* 64-bit source attributes */ +#define KVM_DEV_XIVE_GRP_EAS 4/* 64-bit eas attributes */ +#define KVM_DEV_XIVE_GRP_EQ 5/* 64-bit eq attributes */ + +/* Layout of 64-bit XIVE source attribute values */ +#define KVM_XIVE_LEVEL_SENSITIVE (1ULL << 0) +#define KVM_XIVE_LEVEL_ASSERTED (1ULL << 1) + +/* Layout of 64-bit eas attribute values */ +#define KVM_XIVE_EAS_PRIORITY_SHIFT 0 +#define KVM_XIVE_EAS_PRIORITY_MASK 0x7 +#define KVM_XIVE_EAS_SERVER_SHIFT 3 +#define KVM_XIVE_EAS_SERVER_MASK 0xfffffff8ULL +#define KVM_XIVE_EAS_MASK_SHIFT 32 +#define KVM_XIVE_EAS_MASK_MASK 0x100000000ULL +#define KVM_XIVE_EAS_EISN_SHIFT 33 +#define KVM_XIVE_EAS_EISN_MASK 0xfffffffe00000000ULL + +/* Layout of 64-bit eq attribute */ +#define KVM_XIVE_EQ_PRIORITY_SHIFT 0 +#define KVM_XIVE_EQ_PRIORITY_MASK 0x7 +#define KVM_XIVE_EQ_SERVER_SHIFT 3 +#define KVM_XIVE_EQ_SERVER_MASK 0xfffffff8ULL + +/* Layout of 64-bit eq attribute values */ +struct kvm_ppc_xive_eq { +__u32 flags; +__u32 qsize; +__u64 qpage; +__u32 qtoggle; +__u32 qindex; +}; + +#define KVM_XIVE_EQ_FLAG_ENABLED 0x00000001 +#define KVM_XIVE_EQ_FLAG_ALWAYS_NOTIFY 0x00000002 +#define KVM_XIVE_EQ_FLAG_ESCALATE 0x00000004 +#endif /* __LINUX_KVM_POWERPC_H */
linux-headers/linux/kvm.h +9 -0
@@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef __LINUX_KVM_H#define __LINUX_KVM_H
@@ -965,6 +966,10 @@ struct kvm_ppc_resize_hpt {#define KVM_CAP_COALESCED_PIO 162#define KVM_CAP_HYPERV_ENLIGHTENED_VMCS 163#define KVM_CAP_EXCEPTION_PAYLOAD 164 +#define KVM_CAP_ARM_VM_IPA_SIZE 165 +#define KVM_CAP_MANUAL_DIRTY_LOG_PROTECT 166 +#define KVM_CAP_HYPERV_CPUID 167 +#define KVM_CAP_PPC_IRQ_XIVE 168#ifdef KVM_CAP_IRQ_ROUTING
@@ -1188,6 +1193,8 @@ enum kvm_device_type {#define KVM_DEV_TYPE_ARM_VGIC_V3 KVM_DEV_TYPE_ARM_VGIC_V3 KVM_DEV_TYPE_ARM_VGIC_ITS,#define KVM_DEV_TYPE_ARM_VGIC_ITS KVM_DEV_TYPE_ARM_VGIC_ITS + KVM_DEV_TYPE_XIVE, +#define KVM_DEV_TYPE_XIVE KVM_DEV_TYPE_XIVE KVM_DEV_TYPE_MAX, };
@@ -1305,6 +1312,8 @@ struct kvm_s390_ucas_mapping {#define KVM_GET_DEVICE_ATTR _IOW(KVMIO, 0xe2, struct kvm_device_attr)#define KVM_HAS_DEVICE_ATTR _IOW(KVMIO, 0xe3, struct kvm_device_attr) +#define KVM_DESTROY_DEVICE _IOWR(KVMIO, 0xf0, struct kvm_create_device) +/* * ioctls for vcpu fds */
Signature · 1 line
2.20.1
PATCH 2/13 spapr/xive: add KVM support
In reply to Cédric Le GoaterThis introduces a set of helpers when KVM is in use, which create the KVM XIVE device, initialize the interrupt sources at a KVM level and connect the interrupt presenters to the vCPU.
They also handle the initialization of the TIMA and the source ESB memory regions of the controller. These have a different type under KVM. They are 'ram device' memory mappings, similarly to VFIO, exposed to the guest and the associated VMAs on the host are populated dynamically with the appropriate pages using a fault handler.
--- default-configs/ppc64-softmmu.mak | 1 + include/hw/ppc/spapr_xive.h | 10 ++ include/hw/ppc/xive.h | 22 +++ target/ppc/kvm_ppc.h | 6 + hw/intc/spapr_xive.c | 31 ++-- hw/intc/spapr_xive_kvm.c | 254 ++++++++++++++++++++++++++++++ hw/intc/xive.c | 22 ++- hw/ppc/spapr_irq.c | 11 +- target/ppc/kvm.c | 7 + hw/intc/Makefile.objs | 1 + 10 files changed, 349 insertions(+), 16 deletions(-) create mode 100644 hw/intc/spapr_xive_kvm.c
default-configs/ppc64-softmmu.mak +1 -0
@@ -18,6 +18,7 @@ CONFIG_XICS_SPAPR=$(CONFIG_PSERIES) CONFIG_XICS_KVM=$(call land,$(CONFIG_PSERIES),$(CONFIG_KVM)) CONFIG_XIVE=$(CONFIG_PSERIES) CONFIG_XIVE_SPAPR=$(CONFIG_PSERIES) +CONFIG_XIVE_KVM=$(call land,$(CONFIG_PSERIES),$(CONFIG_KVM)) CONFIG_MEM_DEVICE=y CONFIG_DIMM=y CONFIG_SPAPR_RNG=y
include/hw/ppc/spapr_xive.h +10 -0
@@ -35,6 +35,10 @@ typedef struct sPAPRXive {/* TIMA mapping address */ hwaddr tm_base;MemoryRegion tm_mmio; + +/* KVM support */ +int fd; +void *tm_mmap; } sPAPRXive;bool spapr_xive_irq_claim (sPAPRXive *xive,uint32_t lisn,bool lsi);
@@ -48,5 +52,11 @@ void spapr_dt_xive(sPAPRMachineState *spapr, uint32_t nr_servers, void *fdt, uint32_t phandle);void spapr_xive_set_tctx_os_cam (XiveTCTX *tctx);void spapr_xive_mmio_set_enabled (sPAPRXive *xive,bool enable); +void spapr_xive_map_mmio (sPAPRXive *xive); + +/* +* KVM XIVE device helpers +*/ +void kvmppc_xive_connect (sPAPRXive *xive,Error **errp);#endif /* PPC_SPAPR_XIVE_H */
include/hw/ppc/xive.h +22 -0
@@ -140,6 +140,7 @@#ifndef PPC_XIVE_H #define PPC_XIVE_H +#include "sysemu/kvm.h" #include "hw/qdev-core.h" #include "hw/sysbus.h" #include "hw/ppc/xive_regs.h"
@@ -194,6 +195,9 @@ typedef struct XiveSource {uint32_t esb_shift;MemoryRegion esb_mmio; +/* KVM support */ +void *esb_mmap; +XiveNotifier *xive; } XiveSource;
@@ -421,4 +425,22 @@ static inline uint32_t xive_nvt_cam_line(uint8_t nvt_blk, uint32_t nvt_idx) return (nvt_blk << 19) | nvt_idx; } +/* +* KVM XIVE device helpers +*/ + +/* Keep inlined to discard compile of KVM code sections */ +static inline bool kvmppc_xive_enabled (void ) +{ +if (kvm_enabled ()) { +return machine_kernel_irqchip_allowed (MACHINE (qdev_get_machine ())); + }else { +return false; + } +} + +void kvmppc_xive_source_reset (XiveSource *xsrc,Error **errp); +void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val); +void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp); +#endif /* PPC_XIVE_H */
target/ppc/kvm_ppc.h +6 -0
@@ -59,6 +59,7 @@ bool kvmppc_has_cap_fixup_hcalls(void);bool kvmppc_has_cap_htm (void );bool kvmppc_has_cap_mmu_radix (void );bool kvmppc_has_cap_mmu_hash_v3 (void ); +bool kvmppc_has_cap_xive (void );int kvmppc_get_cap_safe_cache (void );int kvmppc_get_cap_safe_bounds_check (void );int kvmppc_get_cap_safe_indirect_branch (void );
@@ -307,6 +308,11 @@ static inline bool kvmppc_has_cap_mmu_hash_v3(void)return false; } +static inline bool kvmppc_has_cap_xive (void ) +{ +return false; +} +static inline int kvmppc_get_cap_safe_cache (void ) {return 0 ;
hw/intc/spapr_xive.c +21 -10
@@ -172,7 +172,7 @@ void spapr_xive_pic_print_info(sPAPRXive *xive, Monitor *mon) } } -static void spapr_xive_map_mmio (sPAPRXive *xive) +void spapr_xive_map_mmio (sPAPRXive *xive) {sysbus_mmio_map (SYS_BUS_DEVICE (xive),0 , xive->vc_base);sysbus_mmio_map (SYS_BUS_DEVICE (xive),1 , xive->end_base);
@@ -250,6 +250,9 @@ static void spapr_xive_instance_init(Object *obj) TYPE_XIVE_END_SOURCE);object_property_add_child (obj,"end_source" ,OBJECT (&xive->end_source),NULL ); + +/* Not connected to the KVM XIVE device */ + xive->fd =-1 ; }static void spapr_xive_realize (DeviceState *dev,Error **errp)
@@ -304,17 +307,25 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -/* TIMA initialization */ -memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive, -"xive.tima" ,4ull <<TM_SHIFT ); +if (kvmppc_xive_enabled ()) { +kvmppc_xive_connect (xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + }else { +/* TIMA initialization */ +memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive, +"xive.tima" ,4ull <<TM_SHIFT ); -/* Define all XIVE MMIO regions on SysBus */ -sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xsrc->esb_mmio); -sysbus_init_mmio (SYS_BUS_DEVICE (xive), &end_xsrc->esb_mmio); -sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xive->tm_mmio); +/* Define all XIVE MMIO regions on SysBus */ +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xsrc->esb_mmio); +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &end_xsrc->esb_mmio); +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xive->tm_mmio); -/* Map all regions */ -spapr_xive_map_mmio (xive); +/* Map all regions */ +spapr_xive_map_mmio (xive); + }qemu_register_reset (spapr_xive_reset, dev); }
hw/intc/spapr_xive_kvm.c new file +254 -0
254 lines not on this page · raw message
hw/intc/xive.c +19 -3
@@ -15,6 +15,7 @@#include "sysemu/dma.h" #include "hw/qdev-properties.h" #include "monitor/monitor.h" +#include "hw/boards.h" #include "hw/ppc/xive.h" #include "hw/ppc/xive_regs.h"
@@ -493,6 +494,15 @@ static void xive_tctx_realize(DeviceState *dev, Error **errp)return ; } +/* Connect the presenter to the VCPU (required for CPU hotplug) */ +if (kvmppc_xive_enabled ()) { +kvmppc_xive_cpu_connect (tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } +qemu_register_reset (xive_tctx_reset, dev); }
@@ -895,6 +905,10 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); + +if (kvmppc_xive_enabled ()) { +kvmppc_xive_source_reset (xsrc, &error_fatal); + } }static void xive_source_realize (DeviceState *dev,Error **errp)
@@ -928,9 +942,11 @@ static void xive_source_realize(DeviceState *dev, Error **errp) xsrc->status =g_malloc0 (xsrc->nr_irqs); xsrc->lsi_map =bitmap_new (xsrc->nr_irqs); -memory_region_init_io (&xsrc->esb_mmio,OBJECT (xsrc), - &xive_source_esb_ops, xsrc,"xive.esb" , - (1ull << xsrc->esb_shift) * xsrc->nr_irqs); +if (!kvmppc_xive_enabled ()) { +memory_region_init_io (&xsrc->esb_mmio,OBJECT (xsrc), + &xive_source_esb_ops, xsrc,"xive.esb" , + (1ull << xsrc->esb_shift) * xsrc->nr_irqs); + }qemu_register_reset (xive_source_reset, dev); }
hw/ppc/spapr_irq.c +8 -3
@@ -15,6 +15,7 @@#include "hw/ppc/spapr_xive.h" #include "hw/ppc/xics.h" #include "sysemu/kvm.h" +#include "kvm_ppc.h" #include "trace.h"
@@ -266,9 +267,9 @@ static void spapr_irq_init_xive(sPAPRMachineState *spapr, Error **errp)DeviceState *dev;int i; -/* KVM XIVE device not yet available */ if (kvm_enabled ()) { -if (machine_kernel_irqchip_required (machine)) { +if (machine_kernel_irqchip_required (machine) && + !kvmppc_has_cap_xive ()) {error_setg (errp,"kernel_irqchip requested. no KVM XIVE support" );return ; }
@@ -384,7 +385,11 @@ static void spapr_irq_set_irq_xive(void *opaque, int srcno, int val) {sPAPRMachineState *spapr = opaque; -xive_source_set_irq (&spapr->xive->source, srcno, val); +if (kvmppc_xive_enabled ()) { +kvmppc_xive_source_set_irq (&spapr->xive->source, srcno, val); + }else { +xive_source_set_irq (&spapr->xive->source, srcno, val); + } }/*
target/ppc/kvm.c +7 -0
@@ -86,6 +86,7 @@ static int cap_fixup_hcalls;static int cap_htm;/* Hardware transactional memory support */ static int cap_mmu_radix;static int cap_mmu_hash_v3; +static int cap_xive;static int cap_resize_hpt;static int cap_ppc_pvr_compat;static int cap_ppc_safe_cache;
@@ -149,6 +150,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s) cap_htm =kvm_vm_check_extension (s,KVM_CAP_PPC_HTM ); cap_mmu_radix =kvm_vm_check_extension (s,KVM_CAP_PPC_MMU_RADIX ); cap_mmu_hash_v3 =kvm_vm_check_extension (s,KVM_CAP_PPC_MMU_HASH_V3 ); + cap_xive =kvm_vm_check_extension (s,KVM_CAP_PPC_IRQ_XIVE ); cap_resize_hpt =kvm_vm_check_extension (s,KVM_CAP_SPAPR_RESIZE_HPT );kvmppc_get_cpu_characteristics (s); cap_ppc_nested_kvm_hv =kvm_vm_check_extension (s,KVM_CAP_PPC_NESTED_HV );
@@ -2389,6 +2391,11 @@ static int parse_cap_ppc_safe_indirect_branch(struct kvm_ppc_cpu_char c)return 0 ; } +bool kvmppc_has_cap_xive (void ) +{ +return cap_xive; +} +static void kvmppc_get_cpu_characteristics (KVMState *s) {struct kvm_ppc_cpu_char c;
hw/intc/Makefile.objs +1 -0
@@ -39,6 +39,7 @@ obj-$(CONFIG_XICS_SPAPR) += xics_spapr.o obj-$(CONFIG_XICS_KVM) += xics_kvm.o obj-$(CONFIG_XIVE) += xive.o obj-$(CONFIG_XIVE_SPAPR) += spapr_xive.o +obj-$(CONFIG_XIVE_KVM) += spapr_xive_kvm.o obj-$(CONFIG_POWERNV) += xics_pnv.o obj-$(CONFIG_ALLWINNER_A10_PIC) += allwinner-a10-pic.o obj-$(CONFIG_S390_FLIC) += s390_flic.o
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 02/13] spapr/xive: add KVM support
Quoted 11 lines · Cédric Le Goater
This introduces a set of helpers when KVM is in use, which create the KVM XIVE device, initialize the interrupt sources at a KVM level and connect the interrupt presenters to the vCPU.
They also handle the initialization of the TIMA and the source ESB memory regions of the controller. These have a different type under KVM. They are 'ram device' memory mappings, similarly to VFIO, exposed to the guest and the associated VMAs on the host are populated dynamically with the appropriate pages using a fault handler.
Signed-off-by Cédric Le Goater <[email protected]>
Looks fine apart from the details of how the KVM interface works.
Quoted 575 lines
--- default-configs/ppc64-softmmu.mak | 1 + include/hw/ppc/spapr_xive.h | 10 ++ include/hw/ppc/xive.h | 22 +++ target/ppc/kvm_ppc.h | 6 + hw/intc/spapr_xive.c | 31 ++-- hw/intc/spapr_xive_kvm.c | 254 ++++++++++++++++++++++++++++++ hw/intc/xive.c | 22 ++- hw/ppc/spapr_irq.c | 11 +- target/ppc/kvm.c | 7 + hw/intc/Makefile.objs | 1 + 10 files changed, 349 insertions(+), 16 deletions(-) create mode 100644 hw/intc/spapr_xive_kvm.cdefault-configs/ppc64-softmmu.mak +1
-0@@ -18,6 +18,7 @@ CONFIG_XICS_SPAPR=$(CONFIG_PSERIES) CONFIG_XICS_KVM=$(call land,$(CONFIG_PSERIES),$(CONFIG_KVM)) CONFIG_XIVE=$(CONFIG_PSERIES) CONFIG_XIVE_SPAPR=$(CONFIG_PSERIES) +CONFIG_XIVE_KVM=$(call land,$(CONFIG_PSERIES),$(CONFIG_KVM)) CONFIG_MEM_DEVICE=y CONFIG_DIMM=y CONFIG_SPAPR_RNG=yinclude/hw/ppc/spapr_xive.h +10
-0@@ -35,6 +35,10 @@ typedef struct sPAPRXive {/* TIMA mapping address */ hwaddr tm_base;MemoryRegion tm_mmio; + +/* KVM support */ +int fd; +void *tm_mmap; } sPAPRXive;bool spapr_xive_irq_claim (sPAPRXive *xive,uint32_t lisn,bool lsi);@@ -48,5 +52,11 @@ void spapr_dt_xive(sPAPRMachineState *spapr, uint32_t nr_servers, void *fdt, uint32_t phandle);void spapr_xive_set_tctx_os_cam (XiveTCTX *tctx);void spapr_xive_mmio_set_enabled (sPAPRXive *xive,bool enable); +void spapr_xive_map_mmio (sPAPRXive *xive); + +/* +* KVM XIVE device helpers +*/ +void kvmppc_xive_connect (sPAPRXive *xive,Error **errp);#endif /* PPC_SPAPR_XIVE_H */ include/hw/ppc/xive.h +22
-0@@ -140,6 +140,7 @@#ifndef PPC_XIVE_H #define PPC_XIVE_H +#include "sysemu/kvm.h" #include "hw/qdev-core.h" #include "hw/sysbus.h" #include "hw/ppc/xive_regs.h" @@ -194,6 +195,9 @@ typedef struct XiveSource {uint32_t esb_shift;MemoryRegion esb_mmio; +/* KVM support */ +void *esb_mmap; +XiveNotifier *xive; } XiveSource;@@ -421,4 +425,22 @@ static inline uint32_t xive_nvt_cam_line(uint8_t nvt_blk, uint32_t nvt_idx) return (nvt_blk << 19) | nvt_idx; } +/* +* KVM XIVE device helpers +*/ + +/* Keep inlined to discard compile of KVM code sections */ +static inline bool kvmppc_xive_enabled (void ) +{ +if (kvm_enabled ()) { +return machine_kernel_irqchip_allowed (MACHINE (qdev_get_machine ())); + }else { +return false; + } +} + +void kvmppc_xive_source_reset (XiveSource *xsrc,Error **errp); +void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val); +void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp); +#endif /* PPC_XIVE_H */ target/ppc/kvm_ppc.h +6
-0@@ -59,6 +59,7 @@ bool kvmppc_has_cap_fixup_hcalls(void);bool kvmppc_has_cap_htm (void );bool kvmppc_has_cap_mmu_radix (void );bool kvmppc_has_cap_mmu_hash_v3 (void ); +bool kvmppc_has_cap_xive (void );int kvmppc_get_cap_safe_cache (void );int kvmppc_get_cap_safe_bounds_check (void );int kvmppc_get_cap_safe_indirect_branch (void );@@ -307,6 +308,11 @@ static inline bool kvmppc_has_cap_mmu_hash_v3(void)return false; } +static inline bool kvmppc_has_cap_xive (void ) +{ +return false; +} +static inline int kvmppc_get_cap_safe_cache (void ) {return 0 ;hw/intc/spapr_xive.c +21
-10@@ -172,7 +172,7 @@ void spapr_xive_pic_print_info(sPAPRXive *xive, Monitor *mon) } } -static void spapr_xive_map_mmio (sPAPRXive *xive) +void spapr_xive_map_mmio (sPAPRXive *xive) {sysbus_mmio_map (SYS_BUS_DEVICE (xive),0 , xive->vc_base);sysbus_mmio_map (SYS_BUS_DEVICE (xive),1 , xive->end_base);@@ -250,6 +250,9 @@ static void spapr_xive_instance_init(Object *obj) TYPE_XIVE_END_SOURCE);object_property_add_child (obj,"end_source" ,OBJECT (&xive->end_source),NULL ); + +/* Not connected to the KVM XIVE device */ + xive->fd =-1 ; }static void spapr_xive_realize (DeviceState *dev,Error **errp)@@ -304,17 +307,25 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -/* TIMA initialization */ -memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive, -"xive.tima" ,4ull <<TM_SHIFT ); +if (kvmppc_xive_enabled ()) { +kvmppc_xive_connect (xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + }else { +/* TIMA initialization */ +memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive, +"xive.tima" ,4ull <<TM_SHIFT ); -/* Define all XIVE MMIO regions on SysBus */ -sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xsrc->esb_mmio); -sysbus_init_mmio (SYS_BUS_DEVICE (xive), &end_xsrc->esb_mmio); -sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xive->tm_mmio); +/* Define all XIVE MMIO regions on SysBus */ +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xsrc->esb_mmio); +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &end_xsrc->esb_mmio); +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xive->tm_mmio); -/* Map all regions */ -spapr_xive_map_mmio (xive); +/* Map all regions */ +spapr_xive_map_mmio (xive); + }qemu_register_reset (spapr_xive_reset, dev); }hw/intc/spapr_xive_kvm.c new file +254
-0@@ -0,0 +1,254 @@ +/* +* QEMU PowerPC sPAPR XIVE interrupt controller model +* +* Copyright (c) 2017-2019, IBM Corporation. +* +* This code is licensed under the GPL version 2 or later. See the +* COPYING file in the top-level directory. +*/ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "target/ppc/cpu.h" +#include "sysemu/cpus.h" +#include "sysemu/kvm.h" +#include "hw/ppc/spapr.h" +#include "hw/ppc/spapr_xive.h" +#include "hw/ppc/xive.h" +#include "kvm_ppc.h" + +#include <sys/ioctl.h> + +/* +* Helpers for CPU hotplug +* +* TODO: make a common KVMEnabledCPU layer for XICS and XIVE +*/ +typedef struct KVMEnabledCPU { +unsigned long vcpu_id; +QLIST_ENTRY (KVMEnabledCPU) node; +}KVMEnabledCPU ; + +static QLIST_HEAD(, KVMEnabledCPU) + kvm_enabled_cpus =QLIST_HEAD_INITIALIZER (&kvm_enabled_cpus); + +static bool kvm_cpu_is_enabled (CPUState *cs) +{ +KVMEnabledCPU *enabled_cpu; +unsigned long vcpu_id =kvm_arch_vcpu_id (cs); + + QLIST_FOREACH(enabled_cpu, &kvm_enabled_cpus, node) { +if (enabled_cpu->vcpu_id == vcpu_id) { +return true; + } + } +return false; +} + +static void kvm_cpu_enable (CPUState *cs) +{ +KVMEnabledCPU *enabled_cpu; +unsigned long vcpu_id =kvm_arch_vcpu_id (cs); + + enabled_cpu =g_malloc (sizeof (*enabled_cpu)); + enabled_cpu->vcpu_id = vcpu_id; +QLIST_INSERT_HEAD (&kvm_enabled_cpus, enabled_cpu, node); +} + +/* +* XIVE Thread Interrupt Management context (KVM) +*/ + +void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp) +{ +sPAPRXive *xive =SPAPR_MACHINE (qdev_get_machine ())->xive; +unsigned long vcpu_id; +int ret; + +/* Check if CPU was hot unplugged and replugged. */ +if (kvm_cpu_is_enabled (tctx->cs)) { +return ; + } + + vcpu_id =kvm_arch_vcpu_id (tctx->cs); + + ret =kvm_vcpu_enable_cap (tctx->cs,KVM_CAP_PPC_IRQ_XIVE ,0 , xive->fd, + vcpu_id,0 ); +if (ret <0 ) { +error_setg (errp,"XIVE: unable to connect CPU%ld to KVM device: %s" , + vcpu_id,strerror (errno)); +return ; + } + +kvm_cpu_enable (tctx->cs); +} + +/* +* XIVE Interrupt Source (KVM) +*/ + +/* +* At reset, the interrupt sources are simply created and MASKED. We +* only need to inform the KVM XIVE device about their type: LSI or +* MSI. +*/ +void kvmppc_xive_source_reset (XiveSource *xsrc,Error **errp) +{ +sPAPRXive *xive =SPAPR_XIVE (xsrc->xive); +int i; + +for (i =0 ; i < xsrc->nr_irqs; i++) { +Error *local_err =NULL ; +uint64_t state =0 ; + +if (xive_source_irq_is_lsi (xsrc, i)) { + state |=KVM_XIVE_LEVEL_SENSITIVE ; +if (xsrc->status[i] &XIVE_STATUS_ASSERTED ) { + state |=KVM_XIVE_LEVEL_ASSERTED ; + } + } + +kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_SOURCES , i, &state, + true, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } +} + +void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val) +{ +XiveSource *xsrc = opaque; +struct kvm_irq_level args; +int rc; + + args.irq = srcno; +if (!xive_source_irq_is_lsi (xsrc, srcno)) { +if (!val) { +return ; + } + args.level =KVM_INTERRUPT_SET ; + }else { +if (val) { + xsrc->status[srcno] |=XIVE_STATUS_ASSERTED ; + args.level =KVM_INTERRUPT_SET_LEVEL ; + }else { + xsrc->status[srcno] &= ~XIVE_STATUS_ASSERTED ; + args.level =KVM_INTERRUPT_UNSET ; + } + } + rc =kvm_vm_ioctl (kvm_state,KVM_IRQ_LINE , &args); +if (rc <0 ) { +error_report ("XIVE: kvm_irq_line() failed : %s" ,strerror (errno)); + } +} + +/* +* sPAPR XIVE interrupt controller (KVM) +*/ + +static void *kvmppc_xive_mmap (sPAPRXive *xive,int ctrl,size_t len, +Error **errp) +{ +Error *local_err =NULL ; +void *addr; +int fd; + +kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_CTRL , ctrl, &fd, false, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return NULL ; + } + + addr =mmap (NULL , len,PROT_WRITE |PROT_READ ,MAP_SHARED , fd,0 ); +close (fd); +if (addr ==MAP_FAILED ) { +error_setg_errno (errp, errno,"XIVE: unable to set memory mapping" ); +return NULL ; + } + +return addr; +} + +/* +* All the XIVE memory regions are now backed by mappings from the KVM +* XIVE device. +*/ +void kvmppc_xive_connect (sPAPRXive *xive,Error **errp) +{ +XiveSource *xsrc = &xive->source; +XiveENDSource *end_xsrc = &xive->end_source; +Error *local_err =NULL ; +size_t esb_len; +size_t tima_len; + +if (!kvm_enabled () || !kvmppc_has_cap_xive ()) { +error_setg (errp,"IRQ_XIVE capability must be present for KVM" ); +return ; + } + +/* First, create the KVM XIVE device */ + xive->fd =kvm_create_device (kvm_state,KVM_DEV_TYPE_XIVE , false); +if (xive->fd <0 ) { +error_setg_errno (errp, -xive->fd,"XIVE: error creating KVM device" ); +return ; + } + +/* +* Source ESBs KVM mapping +* +* Inform KVM where we will map the ESB pages. This is needed by +* the H_INT_GET_SOURCE_INFO hcall which returns the source +* characteristics, among which the ESB page address. +*/ +kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_CTRL ,KVM_DEV_XIVE_VC_BASE , + &xive->vc_base, true, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + + esb_len = (1ull << xsrc->esb_shift) * xsrc->nr_irqs; + xsrc->esb_mmap =kvmppc_xive_mmap (xive,KVM_DEV_XIVE_GET_ESB_FD , + esb_len, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + +memory_region_init_ram_device_ptr (&xsrc->esb_mmio,OBJECT (xsrc), +"xive.esb" , esb_len, xsrc->esb_mmap); +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xsrc->esb_mmio); + +/* END ESBs mapping (No KVM) */ +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &end_xsrc->esb_mmio); + +/* +* TIMA KVM mapping +* +* We could also inform KVM where the TIMA will be mapped but as +* this is a fixed MMIO address for the system it does not seem +* necessary to provide a KVM ioctl to change it. +*/ + tima_len =4ull <<TM_SHIFT ; + xive->tm_mmap =kvmppc_xive_mmap (xive,KVM_DEV_XIVE_GET_TIMA_FD , + tima_len, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } +memory_region_init_ram_device_ptr (&xive->tm_mmio,OBJECT (xive), +"xive.tima" , tima_len, xive->tm_mmap); +sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xive->tm_mmio); + + kvm_kernel_irqchip = true; + kvm_msi_via_irqfd_allowed = true; + kvm_gsi_direct_mapping = true; + +/* Map all regions */ +spapr_xive_map_mmio (xive); +}hw/intc/xive.c +19
-3@@ -15,6 +15,7 @@#include "sysemu/dma.h" #include "hw/qdev-properties.h" #include "monitor/monitor.h" +#include "hw/boards.h" #include "hw/ppc/xive.h" #include "hw/ppc/xive_regs.h" @@ -493,6 +494,15 @@ static void xive_tctx_realize(DeviceState *dev, Error **errp)return ; } +/* Connect the presenter to the VCPU (required for CPU hotplug) */ +if (kvmppc_xive_enabled ()) { +kvmppc_xive_cpu_connect (tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } +qemu_register_reset (xive_tctx_reset, dev); }@@ -895,6 +905,10 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); + +if (kvmppc_xive_enabled ()) { +kvmppc_xive_source_reset (xsrc, &error_fatal); + } }static void xive_source_realize (DeviceState *dev,Error **errp)@@ -928,9 +942,11 @@ static void xive_source_realize(DeviceState *dev, Error **errp) xsrc->status =g_malloc0 (xsrc->nr_irqs); xsrc->lsi_map =bitmap_new (xsrc->nr_irqs); -memory_region_init_io (&xsrc->esb_mmio,OBJECT (xsrc), - &xive_source_esb_ops, xsrc,"xive.esb" , - (1ull << xsrc->esb_shift) * xsrc->nr_irqs); +if (!kvmppc_xive_enabled ()) { +memory_region_init_io (&xsrc->esb_mmio,OBJECT (xsrc), + &xive_source_esb_ops, xsrc,"xive.esb" , + (1ull << xsrc->esb_shift) * xsrc->nr_irqs); + }qemu_register_reset (xive_source_reset, dev); }hw/ppc/spapr_irq.c +8
-3@@ -15,6 +15,7 @@#include "hw/ppc/spapr_xive.h" #include "hw/ppc/xics.h" #include "sysemu/kvm.h" +#include "kvm_ppc.h" #include "trace.h" @@ -266,9 +267,9 @@ static void spapr_irq_init_xive(sPAPRMachineState *spapr, Error **errp)DeviceState *dev;int i; -/* KVM XIVE device not yet available */ if (kvm_enabled ()) { -if (machine_kernel_irqchip_required (machine)) { +if (machine_kernel_irqchip_required (machine) && + !kvmppc_has_cap_xive ()) {error_setg (errp,"kernel_irqchip requested. no KVM XIVE support" );return ; }@@ -384,7 +385,11 @@ static void spapr_irq_set_irq_xive(void *opaque, int srcno, int val) {sPAPRMachineState *spapr = opaque; -xive_source_set_irq (&spapr->xive->source, srcno, val); +if (kvmppc_xive_enabled ()) { +kvmppc_xive_source_set_irq (&spapr->xive->source, srcno, val); + }else { +xive_source_set_irq (&spapr->xive->source, srcno, val); + } }/* target/ppc/kvm.c +7
-0@@ -86,6 +86,7 @@ static int cap_fixup_hcalls;static int cap_htm;/* Hardware transactional memory support */ static int cap_mmu_radix;static int cap_mmu_hash_v3; +static int cap_xive;static int cap_resize_hpt;static int cap_ppc_pvr_compat;static int cap_ppc_safe_cache;@@ -149,6 +150,7 @@ int kvm_arch_init(MachineState *ms, KVMState *s) cap_htm =kvm_vm_check_extension (s,KVM_CAP_PPC_HTM ); cap_mmu_radix =kvm_vm_check_extension (s,KVM_CAP_PPC_MMU_RADIX ); cap_mmu_hash_v3 =kvm_vm_check_extension (s,KVM_CAP_PPC_MMU_HASH_V3 ); + cap_xive =kvm_vm_check_extension (s,KVM_CAP_PPC_IRQ_XIVE ); cap_resize_hpt =kvm_vm_check_extension (s,KVM_CAP_SPAPR_RESIZE_HPT );kvmppc_get_cpu_characteristics (s); cap_ppc_nested_kvm_hv =kvm_vm_check_extension (s,KVM_CAP_PPC_NESTED_HV );@@ -2389,6 +2391,11 @@ static int parse_cap_ppc_safe_indirect_branch(struct kvm_ppc_cpu_char c)return 0 ; } +bool kvmppc_has_cap_xive (void ) +{ +return cap_xive; +} +static void kvmppc_get_cpu_characteristics (KVMState *s) {struct kvm_ppc_cpu_char c;hw/intc/Makefile.objs +1
-0@@ -39,6 +39,7 @@ obj-$(CONFIG_XICS_SPAPR) += xics_spapr.o obj-$(CONFIG_XICS_KVM) += xics_kvm.o obj-$(CONFIG_XIVE) += xive.o obj-$(CONFIG_XIVE_SPAPR) += spapr_xive.o +obj-$(CONFIG_XIVE_KVM) += spapr_xive_kvm.o obj-$(CONFIG_POWERNV) += xics_pnv.o obj-$(CONFIG_ALLWINNER_A10_PIC) += allwinner-a10-pic.o obj-$(CONFIG_S390_FLIC) += s390_flic.o
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 3/13 spapr/xive: add state synchronization with KVM
In reply to Cédric Le GoaterThis extends the KVM XIVE device backend with 'synchronize_state' methods used to retrieve the state from KVM. The HW state of the sources, the KVM device and the thread interrupt contexts are collected for the monitor usage and also migration.
These get operations rely on their KVM counterpart in the host kernel which acts as a proxy for OPAL, the host firmware. The set operations will be added for migration support later.
--- include/hw/ppc/spapr_xive.h | 9 ++ include/hw/ppc/xive.h | 1 + hw/intc/spapr_xive.c | 24 ++-- hw/intc/spapr_xive_kvm.c | 223 ++++++++++++++++++++++++++++++++++++ hw/intc/xive.c | 10 ++ 5 files changed, 260 insertions(+), 7 deletions(-)
include/hw/ppc/spapr_xive.h +9 -0
@@ -44,6 +44,14 @@ typedef struct sPAPRXive {bool spapr_xive_irq_claim (sPAPRXive *xive,uint32_t lisn,bool lsi);bool spapr_xive_irq_free (sPAPRXive *xive,uint32_t lisn);void spapr_xive_pic_print_info (sPAPRXive *xive,Monitor *mon); +bool spapr_xive_priority_is_reserved (uint8_t priority); + +void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu, +uint8_t *out_nvt_blk,uint32_t *out_nvt_idx); +void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx); +int spapr_xive_target_to_end (uint32_t target,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx);typedef struct sPAPRMachineState sPAPRMachineState ;
@@ -58,5 +66,6 @@ void spapr_xive_map_mmio(sPAPRXive *xive);* KVM XIVE device helpers */ void kvmppc_xive_connect (sPAPRXive *xive,Error **errp); +void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp);#endif /* PPC_SPAPR_XIVE_H */
include/hw/ppc/xive.h +1 -0
@@ -442,5 +442,6 @@ static inline bool kvmppc_xive_enabled(void)void kvmppc_xive_source_reset (XiveSource *xsrc,Error **errp);void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val);void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp); +void kvmppc_xive_cpu_synchronize_state (XiveTCTX *tctx,Error **errp);#endif /* PPC_XIVE_H */
hw/intc/spapr_xive.c +17 -7
@@ -54,8 +54,8 @@ static uint32_t spapr_xive_nvt_to_target(uint8_t nvt_blk, uint32_t nvt_idx)return nvt_idx -SPAPR_XIVE_NVT_BASE ; } -static void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu, -uint8_t *out_nvt_blk,uint32_t *out_nvt_idx) +void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu, +uint8_t *out_nvt_blk,uint32_t *out_nvt_idx) {assert (cpu);
@@ -85,8 +85,8 @@ static int spapr_xive_target_to_nvt(uint32_t target,* sPAPR END indexing uses a simple mapping of the CPU vcpu_id, 8 * priorities per CPU */ -static void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio, -uint8_t *out_end_blk,uint32_t *out_end_idx) +void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx) {assert (cpu);
@@ -99,8 +99,8 @@ static void spapr_xive_cpu_to_end(PowerPCCPU *cpu, uint8_t prio, } } -static int spapr_xive_target_to_end (uint32_t target,uint8_t prio, -uint8_t *out_end_blk,uint32_t *out_end_idx) +int spapr_xive_target_to_end (uint32_t target,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx) {PowerPCCPU *cpu =spapr_find_cpu (target);
@@ -139,6 +139,16 @@ void spapr_xive_pic_print_info(sPAPRXive *xive, Monitor *mon)XiveSource *xsrc = &xive->source;int i; +if (kvmppc_xive_enabled ()) { +Error *local_err =NULL ; + +kvmppc_xive_synchronize_state (xive, &local_err); +if (local_err) { +error_report_err (local_err); +return ; + } + } +monitor_printf (mon," LSIN PQ EISN CPU/PRIO EQ\n" );for (i =0 ; i < xive->nr_irqs; i++) {
@@ -529,7 +539,7 @@ bool spapr_xive_irq_free(sPAPRXive *xive, uint32_t lisn)* interrupts (DD2.X POWER9). So we only allow the guest to use * priorities [0..6]. */ -static bool spapr_xive_priority_is_reserved (uint8_t priority) +bool spapr_xive_priority_is_reserved (uint8_t priority) {switch (priority) { case 0 ... 6:
hw/intc/spapr_xive_kvm.c +223 -0
241 lines not on this page · raw message
hw/intc/xive.c +10 -0
@@ -434,6 +434,16 @@ void xive_tctx_pic_print_info(XiveTCTX *tctx, Monitor *mon)int cpu_index = tctx->cs ? tctx->cs->cpu_index :-1 ;int i; +if (kvmppc_xive_enabled ()) { +Error *local_err =NULL ; + +kvmppc_xive_cpu_synchronize_state (tctx, &local_err); +if (local_err) { +error_report_err (local_err); +return ; + } + } +monitor_printf (mon,"CPU[%04x]: QW NSR CPPR IPB LSMFB ACK# INC AGE PIPR" " W2\n" , cpu_index);
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 03/13] spapr/xive: add state synchronization with KVM
Quoted 10 lines · Cédric Le Goater
This extends the KVM XIVE device backend with '
synchronize_state' methods used to retrieve the state from KVM. The HW state of the sources, the KVM device and the thread interrupt contexts are collected for the monitor usage and also migration.These get operations rely on their KVM counterpart in the host kernel which acts as a proxy for OPAL, the host firmware. The set operations will be added for migration support later.
Signed-off-by Cédric Le Goater <[email protected]>
Quoted 377 lines
--- include/hw/ppc/spapr_xive.h | 9 ++ include/hw/ppc/xive.h | 1 + hw/intc/spapr_xive.c | 24 ++-- hw/intc/spapr_xive_kvm.c | 223 ++++++++++++++++++++++++++++++++++++ hw/intc/xive.c | 10 ++ 5 files changed, 260 insertions(+), 7 deletions(-)include/hw/ppc/spapr_xive.h +9
-0@@ -44,6 +44,14 @@ typedef struct sPAPRXive {bool spapr_xive_irq_claim (sPAPRXive *xive,uint32_t lisn,bool lsi);bool spapr_xive_irq_free (sPAPRXive *xive,uint32_t lisn);void spapr_xive_pic_print_info (sPAPRXive *xive,Monitor *mon); +bool spapr_xive_priority_is_reserved (uint8_t priority); + +void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu, +uint8_t *out_nvt_blk,uint32_t *out_nvt_idx); +void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx); +int spapr_xive_target_to_end (uint32_t target,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx);typedef struct sPAPRMachineState sPAPRMachineState ;@@ -58,5 +66,6 @@ void spapr_xive_map_mmio(sPAPRXive *xive);* KVM XIVE device helpers */ void kvmppc_xive_connect (sPAPRXive *xive,Error **errp); +void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp);#endif /* PPC_SPAPR_XIVE_H */ include/hw/ppc/xive.h +1
-0@@ -442,5 +442,6 @@ static inline bool kvmppc_xive_enabled(void)void kvmppc_xive_source_reset (XiveSource *xsrc,Error **errp);void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val);void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp); +void kvmppc_xive_cpu_synchronize_state (XiveTCTX *tctx,Error **errp);#endif /* PPC_XIVE_H */ hw/intc/spapr_xive.c +17
-7@@ -54,8 +54,8 @@ static uint32_t spapr_xive_nvt_to_target(uint8_t nvt_blk, uint32_t nvt_idx)return nvt_idx -SPAPR_XIVE_NVT_BASE ; } -static void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu, -uint8_t *out_nvt_blk,uint32_t *out_nvt_idx) +void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu, +uint8_t *out_nvt_blk,uint32_t *out_nvt_idx) {assert (cpu);@@ -85,8 +85,8 @@ static int spapr_xive_target_to_nvt(uint32_t target,* sPAPR END indexing uses a simple mapping of the CPU vcpu_id, 8 * priorities per CPU */ -static void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio, -uint8_t *out_end_blk,uint32_t *out_end_idx) +void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx) {assert (cpu);@@ -99,8 +99,8 @@ static void spapr_xive_cpu_to_end(PowerPCCPU *cpu, uint8_t prio, } } -static int spapr_xive_target_to_end (uint32_t target,uint8_t prio, -uint8_t *out_end_blk,uint32_t *out_end_idx) +int spapr_xive_target_to_end (uint32_t target,uint8_t prio, +uint8_t *out_end_blk,uint32_t *out_end_idx) {PowerPCCPU *cpu =spapr_find_cpu (target);@@ -139,6 +139,16 @@ void spapr_xive_pic_print_info(sPAPRXive *xive, Monitor *mon)XiveSource *xsrc = &xive->source;int i; +if (kvmppc_xive_enabled ()) { +Error *local_err =NULL ; + +kvmppc_xive_synchronize_state (xive, &local_err); +if (local_err) { +error_report_err (local_err); +return ; + } + } +monitor_printf (mon," LSIN PQ EISN CPU/PRIO EQ\n" );for (i =0 ; i < xive->nr_irqs; i++) {@@ -529,7 +539,7 @@ bool spapr_xive_irq_free(sPAPRXive *xive, uint32_t lisn)* interrupts (DD2.X POWER9). So we only allow the guest to use * priorities [0..6]. */ -static bool spapr_xive_priority_is_reserved (uint8_t priority) +bool spapr_xive_priority_is_reserved (uint8_t priority) {switch (priority) { case 0 ... 6:hw/intc/spapr_xive_kvm.c +223
-0@@ -60,6 +60,57 @@ static void kvm_cpu_enable(CPUState *cs)/* * XIVE Thread Interrupt Management context (KVM) */ +static void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp) +{ +uint64_t state[4 ] = {0 }; +int ret; + + ret =kvm_get_one_reg (tctx->cs,KVM_REG_PPC_NVT_STATE , state); +if (ret !=0 ) { +error_setg_errno (errp, errno, +"XIVE: could not capture KVM state of CPU %ld" , +kvm_arch_vcpu_id (tctx->cs)); +return ; + } + +/* word0 and word1 of the OS ring. */ + *((uint64_t *) &tctx->regs[TM_QW1_OS ]) = state[0 ]; + +/* +* KVM also returns word2 containing the OS CAM line which is +* interesting to print out in the QEMU monitor. +*/ + *((uint64_t *) &tctx->regs[TM_QW1_OS +TM_WORD2 ]) = state[1 ]; +} + +typedef struct { +XiveTCTX *tctx; +Error *err; +}XiveCpuGetState ; + +static void kvmppc_xive_cpu_do_synchronize_state (CPUState *cpu, +run_on_cpu_data arg) +{ +XiveCpuGetState *s = arg.host_ptr; + +kvmppc_xive_cpu_get_state (s->tctx, &s->err); +} + +void kvmppc_xive_cpu_synchronize_state (XiveTCTX *tctx,Error **errp) +{ +XiveCpuGetState s = { + .tctx = tctx, + .err =NULL , + }; + +run_on_cpu (tctx->cs, kvmppc_xive_cpu_do_synchronize_state, +RUN_ON_CPU_HOST_PTR (&s)); + +if (s.err) { +error_propagate (errp, s.err); +return ; + } +}void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp) {@@ -119,6 +170,34 @@ void kvmppc_xive_source_reset(XiveSource *xsrc, Error **errp) } } +/* +* This is used to perform the magic loads on the ESB pages, described +* in xive.h. +*/ +static uint8_t xive_esb_read (XiveSource *xsrc,int srcno,uint32_t offset) +{ +unsigned long addr = (unsigned long ) xsrc->esb_mmap + +xive_source_esb_mgmt (xsrc, srcno) + offset; + +/* Prevent the compiler from optimizing away the load */ +volatile uint64_t value = *((uint64_t *) addr); + +return be64_to_cpu (value) &0x3 ; +} + +static void kvmppc_xive_source_get_state (XiveSource *xsrc) +{ +int i; + +for (i =0 ; i < xsrc->nr_irqs; i++) { +/* Perform a load without side effect to retrieve the PQ bits */ +uint8_t pq =xive_esb_read (xsrc, i,XIVE_ESB_GET ); + +/* and save PQ locally */ +xive_source_esb_set (xsrc, i, pq); + } +} +void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val) {XiveSource *xsrc = opaque;@@ -149,6 +228,150 @@ void kvmppc_xive_source_set_irq(void *opaque, int srcno, int val)/* * sPAPR XIVE interrupt controller (KVM) */ +static int kvmppc_xive_get_eq_state (sPAPRXive *xive,CPUState *cs,Error **errp) +{ +unsigned long vcpu_id =kvm_arch_vcpu_id (cs); +int ret; +int i; + +for (i =0 ; i <XIVE_PRIORITY_MAX +1 ; i++) { +Error *local_err =NULL ; +struct kvm_ppc_xive_eq kvm_eq = {0 }; +uint64_t kvm_eq_idx; +XiveEND end = {0 }; +uint8_t end_blk, nvt_blk; +uint32_t end_idx, nvt_idx; + +/* Skip priorities reserved for the hypervisor */ +if (spapr_xive_priority_is_reserved (i)) { +continue ; + } + +/* Encode the tuple (server, prio) as a KVM EQ index */ + kvm_eq_idx = i <<KVM_XIVE_EQ_PRIORITY_SHIFT & +KVM_XIVE_EQ_PRIORITY_MASK ; + kvm_eq_idx |= vcpu_id <<KVM_XIVE_EQ_SERVER_SHIFT & +KVM_XIVE_EQ_SERVER_MASK ; + + ret =kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_EQ , kvm_eq_idx, + &kvm_eq, false, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ret; + } + +if (!(kvm_eq.flags &KVM_XIVE_EQ_FLAG_ENABLED )) { +continue ; + } + +/* Update the local END structure with the KVM input */ +if (kvm_eq.flags &KVM_XIVE_EQ_FLAG_ENABLED ) { + end.w0 |=cpu_to_be32 (END_W0_VALID |END_W0_ENQUEUE ); + } +if (kvm_eq.flags &KVM_XIVE_EQ_FLAG_ALWAYS_NOTIFY ) { + end.w0 |=cpu_to_be32 (END_W0_UCOND_NOTIFY ); + } +if (kvm_eq.flags &KVM_XIVE_EQ_FLAG_ESCALATE ) { + end.w0 |=cpu_to_be32 (END_W0_ESCALATE_CTL ); + } + end.w0 |=xive_set_field32 (END_W0_QSIZE ,0ul , kvm_eq.qsize -12 ); + + end.w1 =xive_set_field32 (END_W1_GENERATION ,0ul , kvm_eq.qtoggle) | +xive_set_field32 (END_W1_PAGE_OFF ,0ul , kvm_eq.qindex); + end.w2 =cpu_to_be32 ((kvm_eq.qpage >>32 ) &0x0fffffff ); + end.w3 =cpu_to_be32 (kvm_eq.qpage &0xffffffff ); + end.w4 =0 ; + end.w5 =0 ; + +spapr_xive_cpu_to_nvt (POWERPC_CPU (cs), &nvt_blk, &nvt_idx); + + end.w6 =xive_set_field32 (END_W6_NVT_BLOCK ,0ul , nvt_blk) | +xive_set_field32 (END_W6_NVT_INDEX ,0ul , nvt_idx); + end.w7 =xive_set_field32 (END_W7_F0_PRIORITY ,0ul , i); + +spapr_xive_cpu_to_end (POWERPC_CPU (cs), i, &end_blk, &end_idx); + +assert (end_idx < xive->nr_ends); +memcpy (&xive->endt[end_idx], &end,sizeof (XiveEND)); + } + +return 0 ; +} + +static void kvmppc_xive_get_eas_state (sPAPRXive *xive,Error **errp) +{ +XiveSource *xsrc = &xive->source; +int i; + +for (i =0 ; i < xsrc->nr_irqs; i++) { +XiveEAS *eas = &xive->eat[i]; +XiveEAS new_eas; +uint64_t kvm_eas; +uint8_t priority; +uint32_t server; +uint32_t end_idx; +uint8_t end_blk; +uint32_t eisn; +Error *local_err =NULL ; + +if (!xive_eas_is_valid (eas)) { +continue ; + } + +kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_EAS , i, &kvm_eas, false, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + + priority = (kvm_eas &KVM_XIVE_EAS_PRIORITY_MASK ) >> +KVM_XIVE_EAS_PRIORITY_SHIFT ; + server = (kvm_eas &KVM_XIVE_EAS_SERVER_MASK ) >> +KVM_XIVE_EAS_SERVER_SHIFT ; + eisn = (kvm_eas &KVM_XIVE_EAS_EISN_MASK ) >>KVM_XIVE_EAS_EISN_SHIFT ; + +if (spapr_xive_target_to_end (server, priority, &end_blk, &end_idx)) { +error_setg (errp,"XIVE: invalid tuple CPU %d priority %d" , server, + priority); +return ; + } + + new_eas.w =cpu_to_be64 (EAS_VALID ); +if (kvm_eas &KVM_XIVE_EAS_MASK_MASK ) { + new_eas.w |=cpu_to_be64 (EAS_MASKED ); + } + + new_eas.w =xive_set_field64 (EAS_END_INDEX , new_eas.w, end_idx); + new_eas.w =xive_set_field64 (EAS_END_BLOCK , new_eas.w, end_blk); + new_eas.w =xive_set_field64 (EAS_END_DATA , new_eas.w, eisn); + + *eas = new_eas; + } +} + +void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp) +{ +XiveSource *xsrc = &xive->source; +CPUState *cs; +Error *local_err =NULL ; + +kvmppc_xive_source_get_state (xsrc); + +kvmppc_xive_get_eas_state (xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + +CPU_FOREACH (cs) { +kvmppc_xive_get_eq_state (xive, cs, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } +}static void *kvmppc_xive_mmap (sPAPRXive *xive,int ctrl,size_t len,Error **errp)hw/intc/xive.c +10
-0@@ -434,6 +434,16 @@ void xive_tctx_pic_print_info(XiveTCTX *tctx, Monitor *mon)int cpu_index = tctx->cs ? tctx->cs->cpu_index :-1 ;int i; +if (kvmppc_xive_enabled ()) { +Error *local_err =NULL ; + +kvmppc_xive_cpu_synchronize_state (tctx, &local_err); +if (local_err) { +error_report_err (local_err); +return ; + } + } +monitor_printf (mon,"CPU[%04x]: QW NSR CPPR IPB LSMFB ACK# INC AGE PIPR" " W2\n" , cpu_index);
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 4/13 spapr/xive: introduce a VM state change handler
In reply to Cédric Le GoaterThis handler is in charge of stabilizing the flow of event notifications in the XIVE controller before migrating a guest. This is a requirement before transferring the guest EQ pages to a destination.
When the VM is stopped, the handler masks the sources (PQ=01) to stop the flow of events and saves their previous state. The XIVE controller is then synced through KVM to flush any in-flight event notification and to stabilize the EQs. At this stage, the EQ pages are marked dirty to make sure the EQ pages are transferred if a migration sequence is in progress.
The previous configuration of the sources is restored when the VM resumes, after a migration or a stop.
--- include/hw/ppc/spapr_xive.h | 1 + hw/intc/spapr_xive_kvm.c | 111 +++++++++++++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 1 deletion(-)
include/hw/ppc/spapr_xive.h +1 -0
@@ -39,6 +39,7 @@ typedef struct sPAPRXive {/* KVM support */ int fd;void *tm_mmap; +VMChangeStateEntry *change; } sPAPRXive;bool spapr_xive_irq_claim (sPAPRXive *xive,uint32_t lisn,bool lsi);
hw/intc/spapr_xive_kvm.c +110 -1
129 lines not on this page · raw message
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 04/13] spapr/xive: introduce a VM state change handler
Quoted 11 lines · Cédric Le Goater
This handler is in charge of stabilizing the flow of event notifications in the XIVE controller before migrating a guest. This is a requirement before transferring the guest EQ pages to a destination.
When the VM is stopped, the handler masks the sources (PQ=01) to stop the flow of events and saves their previous state. The XIVE controller is then synced through KVM to flush any in-flight event notification and to stabilize the EQs. At this stage, the EQ pages are marked dirty to make sure the EQ pages are transferred if a migration sequence is in progress.
Quoted 4 lines
The previous configuration of the sources is restored when the VM resumes, after a migration or a stop.
Signed-off-by Cédric Le Goater <[email protected]>
Looks fine apart from interface details discussed on the kernel series.
Quoted 152 lines
--- include/hw/ppc/spapr_xive.h | 1 + hw/intc/spapr_xive_kvm.c | 111 +++++++++++++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 1 deletion(-)include/hw/ppc/spapr_xive.h +1
-0@@ -39,6 +39,7 @@ typedef struct sPAPRXive {/* KVM support */ int fd;void *tm_mmap; +VMChangeStateEntry *change; } sPAPRXive;bool spapr_xive_irq_claim (sPAPRXive *xive,uint32_t lisn,bool lsi);hw/intc/spapr_xive_kvm.c +110
-1@@ -350,13 +350,119 @@ static void kvmppc_xive_get_eas_state(sPAPRXive *xive, Error **errp) } } +/* +* Sync the XIVE controller through KVM to flush any in-flight event +* notification and stabilize the EQs. +*/ +static void kvmppc_xive_sync_all (sPAPRXive *xive,Error **errp) +{ +XiveSource *xsrc = &xive->source; +Error *local_err =NULL ; +int i; + +/* Sync the KVM source. This reaches the XIVE HW through OPAL */ +for (i =0 ; i < xsrc->nr_irqs; i++) { +XiveEAS *eas = &xive->eat[i]; + +if (!xive_eas_is_valid (eas)) { +continue ; + } + +kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_SYNC , i,NULL , true, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } +} + +/* +* The primary goal of the XIVE VM change handler is to mark the EQ +* pages dirty when all XIVE event notifications have stopped. +* +* Whenever the VM is stopped, the VM change handler masks the sources +* (PQ=01) to stop the flow of events and saves the previous state in +* anticipation of a migration. The XIVE controller is then synced +* through KVM to flush any in-flight event notification and stabilize +* the EQs. +* +* At this stage, we can mark the EQ page dirty and let a migration +* sequence transfer the EQ pages to the destination, which is done +* just after the stop state. +* +* The previous configuration of the sources is restored when the VM +* runs again. +*/ +static void kvmppc_xive_change_state_handler (void *opaque,int running, +RunState state) +{ +sPAPRXive *xive = opaque; +XiveSource *xsrc = &xive->source; +Error *local_err =NULL ; +int i; + +/* +* Restore the sources to their initial state. This is called when +* the VM resumes after a stop or a migration. +*/ +if (running) { +for (i =0 ; i < xsrc->nr_irqs; i++) { +uint8_t pq =xive_source_esb_get (xsrc, i); +if (xive_esb_read (xsrc, i,XIVE_ESB_SET_PQ_00 + (pq <<8 )) !=0x1 ) { +error_report ("XIVE: IRQ %d has an invalid state" , i); + } + } + +return ; + } + +/* +* Mask the sources, to stop the flow of event notifications, and +* save the PQs locally in the XiveSource object. The XiveSource +* state will be collected later on by its vmstate handler if a +* migration is in progress. +*/ +for (i =0 ; i < xsrc->nr_irqs; i++) { +uint8_t pq =xive_esb_read (xsrc, i,XIVE_ESB_SET_PQ_01 ); +xive_source_esb_set (xsrc, i, pq); + } + +/* +* Sync the XIVE controller in KVM, to flush in-flight event +* notification that should be enqueued in the EQs. +*/ +kvmppc_xive_sync_all (xive, &local_err); +if (local_err) { +error_report_err (local_err); +return ; + } + +/* +* Mark the XIVE EQ pages dirty to collect all updates. +*/ +kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_CTRL , +KVM_DEV_XIVE_SAVE_EQ_PAGES ,NULL , true, &local_err); +if (local_err) { +error_report_err (local_err); + } +} +void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp) {XiveSource *xsrc = &xive->source;CPUState *cs;Error *local_err =NULL ; -kvmppc_xive_source_get_state (xsrc); +/* +* When the VM is stopped, the sources are masked and the previous +* state is saved in anticipation of a migration. We should not +* synchronize the source state in that case else we will override +* the saved state. +*/ +if (runstate_is_running ()) { +kvmppc_xive_source_get_state (xsrc); + }kvmppc_xive_get_eas_state (xive, &local_err);if (local_err) {@@ -468,6 +574,9 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) "xive.tima", tima_len, xive->tm_mmap);sysbus_init_mmio (SYS_BUS_DEVICE (xive), &xive->tm_mmio); + xive->change =qemu_add_vm_change_state_handler ( + kvmppc_xive_change_state_handler, xive); + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 5/13 spapr/xive: add migration support for KVM
In reply to Cédric Le GoaterWhen the VM is stopped, the VM state handler stabilizes the XIVE IC and marks the EQ pages dirty. These are then transferred to destination before the transfer of the device vmstates starts.
The sPAPRXive interrupt controller model captures the XIVE internal tables, EAT and ENDT and the XiveTCTX model does the same for the thread interrupt context registers.
At restart, the sPAPRXive 'post_load' method restores all the XIVE states. It is called by the sPAPR machine 'post_load' method, when all XIVE states have been transferred and loaded.
Finally, the source states are restored in the VM change state handler when the machine reaches the running state.
--- include/hw/ppc/spapr_xive.h | 5 + include/hw/ppc/xive.h | 1 + hw/intc/spapr_xive.c | 34 +++++++ hw/intc/spapr_xive_kvm.c | 187 +++++++++++++++++++++++++++++++++++- hw/intc/xive.c | 17 ++++ hw/ppc/spapr_irq.c | 2 +- 6 files changed, 244 insertions(+), 2 deletions(-)
include/hw/ppc/spapr_xive.h +5 -0
@@ -46,6 +46,7 @@ bool spapr_xive_irq_claim(sPAPRXive *xive, uint32_t lisn, bool lsi);bool spapr_xive_irq_free (sPAPRXive *xive,uint32_t lisn);void spapr_xive_pic_print_info (sPAPRXive *xive,Monitor *mon);bool spapr_xive_priority_is_reserved (uint8_t priority); +int spapr_xive_post_load (sPAPRXive *xive,int version_id);void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu,uint8_t *out_nvt_blk,uint32_t *out_nvt_idx);
@@ -53,6 +54,8 @@ void spapr_xive_cpu_to_end(PowerPCCPU *cpu, uint8_t prio, uint8_t *out_end_blk, uint32_t *out_end_idx);int spapr_xive_target_to_end (uint32_t target,uint8_t prio,uint8_t *out_end_blk,uint32_t *out_end_idx); +int spapr_xive_end_to_target (uint8_t end_blk,uint32_t end_idx, +uint32_t *out_server,uint8_t *out_prio);typedef struct sPAPRMachineState sPAPRMachineState ;
@@ -68,5 +71,7 @@ void spapr_xive_map_mmio(sPAPRXive *xive);*/ void kvmppc_xive_connect (sPAPRXive *xive,Error **errp);void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp); +int kvmppc_xive_pre_save (sPAPRXive *xive); +int kvmppc_xive_post_load (sPAPRXive *xive,int version_id);#endif /* PPC_SPAPR_XIVE_H */
include/hw/ppc/xive.h +1 -0
@@ -443,5 +443,6 @@ void kvmppc_xive_source_reset(XiveSource *xsrc, Error **errp);void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val);void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp);void kvmppc_xive_cpu_synchronize_state (XiveTCTX *tctx,Error **errp); +void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp);#endif /* PPC_XIVE_H */
hw/intc/spapr_xive.c +34 -0
@@ -85,6 +85,19 @@ static int spapr_xive_target_to_nvt(uint32_t target,* sPAPR END indexing uses a simple mapping of the CPU vcpu_id, 8 * priorities per CPU */ +int spapr_xive_end_to_target (uint8_t end_blk,uint32_t end_idx, +uint32_t *out_server,uint8_t *out_prio) +{ +if (out_server) { + *out_server = end_idx >>3 ; + } + +if (out_prio) { + *out_prio = end_idx &0x7 ; + } +return 0 ; +} +void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio,uint8_t *out_end_blk,uint32_t *out_end_idx) {
@@ -438,10 +451,31 @@ static const VMStateDescription vmstate_spapr_xive_eas = { }, }; +static int vmstate_spapr_xive_pre_save (void *opaque) +{ +if (kvmppc_xive_enabled ()) { +return kvmppc_xive_pre_save (SPAPR_XIVE (opaque)); + } + +return 0 ; +} + +/* Called by the sPAPR machine 'post_load' method */ +int spapr_xive_post_load (sPAPRXive *xive,int version_id) +{ +if (kvmppc_xive_enabled ()) { +return kvmppc_xive_post_load (xive, version_id); + } + +return 0 ; +} +static const VMStateDescription vmstate_spapr_xive = { .name =TYPE_SPAPR_XIVE , .version_id =1 , .minimum_version_id =1 , + .pre_save = vmstate_spapr_xive_pre_save, + .post_load =NULL ,/* handled at the machine level */ .fields = (VMStateField []) {VMSTATE_UINT32_EQUAL (nr_irqs, sPAPRXive,NULL ), VMSTATE_STRUCT_VARRAY_POINTER_UINT32(eat, sPAPRXive, nr_irqs,
hw/intc/spapr_xive_kvm.c +186 -1
211 lines not on this page · raw message
hw/intc/xive.c +17 -0
@@ -521,10 +521,27 @@ static void xive_tctx_unrealize(DeviceState *dev, Error **errp)qemu_unregister_reset (xive_tctx_reset, dev); } +static int vmstate_xive_tctx_pre_save (void *opaque) +{ +Error *local_err =NULL ; + +if (kvmppc_xive_enabled ()) { +kvmppc_xive_cpu_get_state (XIVE_TCTX (opaque), &local_err); +if (local_err) { +error_report_err (local_err); +return -1 ; + } + } + +return 0 ; +} +static const VMStateDescription vmstate_xive_tctx = { .name =TYPE_XIVE_TCTX , .version_id =1 , .minimum_version_id =1 , + .pre_save = vmstate_xive_tctx_pre_save, + .post_load =NULL ,/* handled by the sPAPRxive model */ .fields = (VMStateField []) {VMSTATE_BUFFER (regs, XiveTCTX),VMSTATE_END_OF_LIST ()
hw/ppc/spapr_irq.c +1 -1
@@ -363,7 +363,7 @@ static void spapr_irq_cpu_intc_create_xive(sPAPRMachineState *spapr,static int spapr_irq_post_load_xive (sPAPRMachineState *spapr,int version_id) { -return 0 ; +return spapr_xive_post_load (spapr->xive, version_id); }static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp)
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 05/13] spapr/xive: add migration support for KVM
Quoted 8 lines · Cédric Le Goater
When the VM is stopped, the VM state handler stabilizes the XIVE IC and marks the EQ pages dirty. These are then transferred to destination before the transfer of the device vmstates starts.
The sPAPRXive interrupt controller model captures the XIVE internal tables, EAT and ENDT and the XiveTCTX model does the same for the thread interrupt context registers.
Quoted 8 lines
At restart, the sPAPRXive '
post_load' method restores all the XIVE states. It is called by the sPAPR machine 'post_load' method, when all XIVE states have been transferred and loaded.Finally, the source states are restored in the VM change state handler when the machine reaches the running state.
Signed-off-by Cédric Le Goater <[email protected]>
Looks find modulo possible changes in the KVM interface.
Quoted 369 lines
--- include/hw/ppc/spapr_xive.h | 5 + include/hw/ppc/xive.h | 1 + hw/intc/spapr_xive.c | 34 +++++++ hw/intc/spapr_xive_kvm.c | 187 +++++++++++++++++++++++++++++++++++- hw/intc/xive.c | 17 ++++ hw/ppc/spapr_irq.c | 2 +- 6 files changed, 244 insertions(+), 2 deletions(-)include/hw/ppc/spapr_xive.h +5
-0@@ -46,6 +46,7 @@ bool spapr_xive_irq_claim(sPAPRXive *xive, uint32_t lisn, bool lsi);bool spapr_xive_irq_free (sPAPRXive *xive,uint32_t lisn);void spapr_xive_pic_print_info (sPAPRXive *xive,Monitor *mon);bool spapr_xive_priority_is_reserved (uint8_t priority); +int spapr_xive_post_load (sPAPRXive *xive,int version_id);void spapr_xive_cpu_to_nvt (PowerPCCPU *cpu,uint8_t *out_nvt_blk,uint32_t *out_nvt_idx);@@ -53,6 +54,8 @@ void spapr_xive_cpu_to_end(PowerPCCPU *cpu, uint8_t prio, uint8_t *out_end_blk, uint32_t *out_end_idx);int spapr_xive_target_to_end (uint32_t target,uint8_t prio,uint8_t *out_end_blk,uint32_t *out_end_idx); +int spapr_xive_end_to_target (uint8_t end_blk,uint32_t end_idx, +uint32_t *out_server,uint8_t *out_prio);typedef struct sPAPRMachineState sPAPRMachineState ;@@ -68,5 +71,7 @@ void spapr_xive_map_mmio(sPAPRXive *xive);*/ void kvmppc_xive_connect (sPAPRXive *xive,Error **errp);void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp); +int kvmppc_xive_pre_save (sPAPRXive *xive); +int kvmppc_xive_post_load (sPAPRXive *xive,int version_id);#endif /* PPC_SPAPR_XIVE_H */ include/hw/ppc/xive.h +1
-0@@ -443,5 +443,6 @@ void kvmppc_xive_source_reset(XiveSource *xsrc, Error **errp);void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val);void kvmppc_xive_cpu_connect (XiveTCTX *tctx,Error **errp);void kvmppc_xive_cpu_synchronize_state (XiveTCTX *tctx,Error **errp); +void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp);#endif /* PPC_XIVE_H */ hw/intc/spapr_xive.c +34
-0@@ -85,6 +85,19 @@ static int spapr_xive_target_to_nvt(uint32_t target,* sPAPR END indexing uses a simple mapping of the CPU vcpu_id, 8 * priorities per CPU */ +int spapr_xive_end_to_target (uint8_t end_blk,uint32_t end_idx, +uint32_t *out_server,uint8_t *out_prio) +{ +if (out_server) { + *out_server = end_idx >>3 ; + } + +if (out_prio) { + *out_prio = end_idx &0x7 ; + } +return 0 ; +} +void spapr_xive_cpu_to_end (PowerPCCPU *cpu,uint8_t prio,uint8_t *out_end_blk,uint32_t *out_end_idx) {@@ -438,10 +451,31 @@ static const VMStateDescription vmstate_spapr_xive_eas = { }, }; +static int vmstate_spapr_xive_pre_save (void *opaque) +{ +if (kvmppc_xive_enabled ()) { +return kvmppc_xive_pre_save (SPAPR_XIVE (opaque)); + } + +return 0 ; +} + +/* Called by the sPAPR machine 'post_load' method */ +int spapr_xive_post_load (sPAPRXive *xive,int version_id) +{ +if (kvmppc_xive_enabled ()) { +return kvmppc_xive_post_load (xive, version_id); + } + +return 0 ; +} +static const VMStateDescription vmstate_spapr_xive = { .name =TYPE_SPAPR_XIVE , .version_id =1 , .minimum_version_id =1 , + .pre_save = vmstate_spapr_xive_pre_save, + .post_load =NULL ,/* handled at the machine level */ .fields = (VMStateField []) {VMSTATE_UINT32_EQUAL (nr_irqs, sPAPRXive,NULL ), VMSTATE_STRUCT_VARRAY_POINTER_UINT32(eat, sPAPRXive, nr_irqs,hw/intc/spapr_xive_kvm.c +186
-1@@ -60,7 +60,30 @@ static void kvm_cpu_enable(CPUState *cs)/* * XIVE Thread Interrupt Management context (KVM) */ -static void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp) + +static void kvmppc_xive_cpu_set_state (XiveTCTX *tctx,Error **errp) +{ +uint64_t state[4 ]; +int ret; + +/* word0 and word1 of the OS ring. */ + state[0 ] = *((uint64_t *) &tctx->regs[TM_QW1_OS ]); + +/* +* OS CAM line. Used by KVM to print out the VP identifier. This +* is for debug only. +*/ + state[1 ] = *((uint64_t *) &tctx->regs[TM_QW1_OS +TM_WORD2 ]); + + ret =kvm_set_one_reg (tctx->cs,KVM_REG_PPC_NVT_STATE , state); +if (ret !=0 ) { +error_setg_errno (errp, errno, +"XIVE: could not restore KVM state of CPU %ld" , +kvm_arch_vcpu_id (tctx->cs)); + } +} + +void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp) {uint64_t state[4 ] = {0 };int ret;@@ -228,6 +251,58 @@ void kvmppc_xive_source_set_irq(void *opaque, int srcno, int val)/* * sPAPR XIVE interrupt controller (KVM) */ +static int kvmppc_xive_set_eq_state (sPAPRXive *xive,CPUState *cs,Error **errp) +{ +unsigned long vcpu_id =kvm_arch_vcpu_id (cs); +int ret; +int i; + +for (i =0 ; i <XIVE_PRIORITY_MAX +1 ; i++) { +Error *local_err =NULL ; +XiveEND *end; +uint8_t end_blk; +uint32_t end_idx; +struct kvm_ppc_xive_eq kvm_eq = {0 }; +uint64_t kvm_eq_idx; + +if (spapr_xive_priority_is_reserved (i)) { +continue ; + } + +spapr_xive_cpu_to_end (POWERPC_CPU (cs), i, &end_blk, &end_idx); + +assert (end_idx < xive->nr_ends); + end = &xive->endt[end_idx]; + +if (!xive_end_is_valid (end)) { +continue ; + } + +/* Build the KVM state from the local END structure */ + kvm_eq.flags =KVM_XIVE_EQ_FLAG_ALWAYS_NOTIFY ; + kvm_eq.qsize =xive_get_field32 (END_W0_QSIZE , end->w0) +12 ; + kvm_eq.qpage = (uint64_t )be32_to_cpu (end->w2 &0x0fffffff ) <<32 | +be32_to_cpu (end->w3); + kvm_eq.qtoggle =xive_get_field32 (END_W1_GENERATION , end->w1); + kvm_eq.qindex =xive_get_field32 (END_W1_PAGE_OFF , end->w1); + +/* Encode the tuple (server, prio) as a KVM EQ index */ + kvm_eq_idx = i <<KVM_XIVE_EQ_PRIORITY_SHIFT & +KVM_XIVE_EQ_PRIORITY_MASK ; + kvm_eq_idx |= vcpu_id <<KVM_XIVE_EQ_SERVER_SHIFT & +KVM_XIVE_EQ_SERVER_MASK ; + + ret =kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_EQ , kvm_eq_idx, + &kvm_eq, true, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ret; + } + } + +return 0 ; +} +static int kvmppc_xive_get_eq_state (sPAPRXive *xive,CPUState *cs,Error **errp) {unsigned long vcpu_id =kvm_arch_vcpu_id (cs);@@ -298,6 +373,48 @@ static int kvmppc_xive_get_eq_state(sPAPRXive *xive, CPUState *cs, Error **errp)return 0 ; } +static void kvmppc_xive_set_eas_state (sPAPRXive *xive,Error **errp) +{ +XiveSource *xsrc = &xive->source; +int i; + +for (i =0 ; i < xsrc->nr_irqs; i++) { +XiveEAS *eas = &xive->eat[i]; +uint32_t end_idx; +uint32_t end_blk; +uint32_t eisn; +uint8_t priority; +uint32_t server; +uint64_t kvm_eas; +Error *local_err =NULL ; + +/* No need to set MASKED EAS, this is the default state after reset */ +if (!xive_eas_is_valid (eas) ||xive_eas_is_masked (eas)) { +continue ; + } + + end_idx =xive_get_field64 (EAS_END_INDEX , eas->w); + end_blk =xive_get_field64 (EAS_END_BLOCK , eas->w); + eisn =xive_get_field64 (EAS_END_DATA , eas->w); + +spapr_xive_end_to_target (end_blk, end_idx, &server, &priority); + + kvm_eas = priority <<KVM_XIVE_EAS_PRIORITY_SHIFT & +KVM_XIVE_EAS_PRIORITY_MASK ; + kvm_eas |= server <<KVM_XIVE_EAS_SERVER_SHIFT & +KVM_XIVE_EAS_SERVER_MASK ; + kvm_eas |= ((uint64_t )eisn <<KVM_XIVE_EAS_EISN_SHIFT ) & +KVM_XIVE_EAS_EISN_MASK ; + +kvm_device_access (xive->fd,KVM_DEV_XIVE_GRP_EAS , i, &kvm_eas, true, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } +} +static void kvmppc_xive_get_eas_state (sPAPRXive *xive,Error **errp) {XiveSource *xsrc = &xive->source;@@ -448,6 +565,74 @@ static void kvmppc_xive_change_state_handler(void *opaque, int running, } } +int kvmppc_xive_pre_save (sPAPRXive *xive) +{ +Error *local_err =NULL ; +CPUState *cs; + +/* Grab the EAT */ +kvmppc_xive_get_eas_state (xive, &local_err); +if (local_err) { +error_report_err (local_err); +return -1 ; + } + +/* +* Grab the ENDT. The EQ index and the toggle bit are what we want +* to capture. +*/ +CPU_FOREACH (cs) { +kvmppc_xive_get_eq_state (xive, cs, &local_err); +if (local_err) { +error_report_err (local_err); +return -1 ; + } + } + +return 0 ; +} + +/* +* The sPAPRXive 'post_load' method is called by the sPAPR machine +* 'post_load' method, when all XIVE states have been transferred and +* loaded. +*/ +int kvmppc_xive_post_load (sPAPRXive *xive,int version_id) +{ +Error *local_err =NULL ; +CPUState *cs; + +/* Restore the ENDT first. The targetting depends on it. */ +CPU_FOREACH (cs) { +kvmppc_xive_set_eq_state (xive, cs, &local_err); +if (local_err) { +error_report_err (local_err); +return -1 ; + } + } + +/* Restore the EAT */ +kvmppc_xive_set_eas_state (xive, &local_err); +if (local_err) { +error_report_err (local_err); +return -1 ; + } + +/* Restore the thread interrupt contexts */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_set_state (cpu->tctx, &local_err); +if (local_err) { +error_report_err (local_err); +return -1 ; + } + } + +/* The source states will be restored when the machine starts running */ +return 0 ; +} +void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp) {XiveSource *xsrc = &xive->source;hw/intc/xive.c +17
-0@@ -521,10 +521,27 @@ static void xive_tctx_unrealize(DeviceState *dev, Error **errp)qemu_unregister_reset (xive_tctx_reset, dev); } +static int vmstate_xive_tctx_pre_save (void *opaque) +{ +Error *local_err =NULL ; + +if (kvmppc_xive_enabled ()) { +kvmppc_xive_cpu_get_state (XIVE_TCTX (opaque), &local_err); +if (local_err) { +error_report_err (local_err); +return -1 ; + } + } + +return 0 ; +} +static const VMStateDescription vmstate_xive_tctx = { .name =TYPE_XIVE_TCTX , .version_id =1 , .minimum_version_id =1 , + .pre_save = vmstate_xive_tctx_pre_save, + .post_load =NULL ,/* handled by the sPAPRxive model */ .fields = (VMStateField []) {VMSTATE_BUFFER (regs, XiveTCTX),VMSTATE_END_OF_LIST ()hw/ppc/spapr_irq.c +1
-1@@ -363,7 +363,7 @@ static void spapr_irq_cpu_intc_create_xive(sPAPRMachineState *spapr,static int spapr_irq_post_load_xive (sPAPRMachineState *spapr,int version_id) { -return 0 ; +return spapr_xive_post_load (spapr->xive, version_id); }static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp)
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 6/13 spapr/xive: fix migration of the XiveTCTX under TCG
In reply to Cédric Le GoaterWhen the thread interrupt management state is retrieved from the KVM VCPU, word2 is saved under the QEMU XIVE thread context to print out the OS CAM line under the QEMU monitor.
This breaks the migration of a TCG guest (and with KVM when kernel_irqchip=off) because the matching algorithm of the presenter relies on the OS CAM value. Fix with an extra reset of the thread contexts to restore the expected value.
--- hw/ppc/spapr_irq.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)
hw/ppc/spapr_irq.c +25 -1
@@ -363,7 +363,31 @@ static void spapr_irq_cpu_intc_create_xive(sPAPRMachineState *spapr,static int spapr_irq_post_load_xive (sPAPRMachineState *spapr,int version_id) { -return spapr_xive_post_load (spapr->xive, version_id); +CPUState *cs; +int ret; + + ret =spapr_xive_post_load (spapr->xive, version_id); +if (ret) { +return ret; + } + +/* +* When the states are collected from the KVM XIVE device, word2 +* of the XiveTCTX is set to print out the OS CAM line under the +* QEMU monitor. +* +* This breaks the migration on a TCG guest (or on KVM with +* kernel_irqchip=off) because the matching algorithm of the +* presenter relies on the OS CAM value. Fix with an extra reset +* of the thread contexts to restore the expected value. +*/ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +/* (TCG) Set the OS CAM line of the thread interrupt context. */ +spapr_xive_set_tctx_os_cam (cpu->tctx); + } +return 0 ; }static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp)
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 06/13] spapr/xive: fix migration of the XiveTCTX under TCG
Quoted 10 lines · Cédric Le Goater
When the thread interrupt management state is retrieved from the KVM VCPU, word2 is saved under the QEMU XIVE thread context to print out the OS CAM line under the QEMU monitor.
This breaks the migration of a TCG guest (and with KVM when kernel_irqchip=off) because the matching algorithm of the presenter relies on the OS CAM value. Fix with an extra reset of the thread contexts to restore the expected value.
Signed-off-by Cédric Le Goater <[email protected]>
Why is the CAM value you get from KVM different from the one you expect in qemu?
Quoted 41 lines
--- hw/ppc/spapr_irq.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)hw/ppc/spapr_irq.c +25
-1@@ -363,7 +363,31 @@ static void spapr_irq_cpu_intc_create_xive(sPAPRMachineState *spapr,static int spapr_irq_post_load_xive (sPAPRMachineState *spapr,int version_id) { -return spapr_xive_post_load (spapr->xive, version_id); +CPUState *cs; +int ret; + + ret =spapr_xive_post_load (spapr->xive, version_id); +if (ret) { +return ret; + } + +/* +* When the states are collected from the KVM XIVE device, word2 +* of the XiveTCTX is set to print out the OS CAM line under the +* QEMU monitor. +* +* This breaks the migration on a TCG guest (or on KVM with +* kernel_irqchip=off) because the matching algorithm of the +* presenter relies on the OS CAM value. Fix with an extra reset +* of the thread contexts to restore the expected value. +*/ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +/* (TCG) Set the OS CAM line of the thread interrupt context. */ +spapr_xive_set_tctx_os_cam (cpu->tctx); + } +return 0 ; }static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp)
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 06/13] spapr/xive: fix migration of the XiveTCTX under TCG
Quoted 10 lines · David Gibson
Quoted 9 lines · Cédric Le Goater
When the thread interrupt management state is retrieved from the KVM VCPU, word2 is saved under the QEMU XIVE thread context to print out the OS CAM line under the QEMU monitor.
This breaks the migration of a TCG guest (and with KVM when kernel_irqchip=off) because the matching algorithm of the presenter relies on the OS CAM value. Fix with an extra reset of the thread contexts to restore the expected value.
Quoted 4 lines
Quoted 1 line
Signed-off-by Cédric Le Goater <[email protected]>Why is the CAM value you get from KVM different from the one you expect in qemu?
An NVT base identifier is allocated for each VM at the OPAL level and each vCPU getd an increment of this value. It is pushed in the OS CAM line when the vCPU is scheduled to run.
KVM XIVE names this identifier a VP id.
The QEMU emulation of XIVE uses a fixed value for the NVT base identifier.
C.
Quoted 42 lines
Quoted 41 lines
--- hw/ppc/spapr_irq.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-)hw/ppc/spapr_irq.c +25
-1@@ -363,7 +363,31 @@ static void spapr_irq_cpu_intc_create_xive(sPAPRMachineState *spapr,static int spapr_irq_post_load_xive (sPAPRMachineState *spapr,int version_id) { -return spapr_xive_post_load (spapr->xive, version_id); +CPUState *cs; +int ret; + + ret =spapr_xive_post_load (spapr->xive, version_id); +if (ret) { +return ret; + } + +/* +* When the states are collected from the KVM XIVE device, word2 +* of the XiveTCTX is set to print out the OS CAM line under the +* QEMU monitor. +* +* This breaks the migration on a TCG guest (or on KVM with +* kernel_irqchip=off) because the matching algorithm of the +* presenter relies on the OS CAM value. Fix with an extra reset +* of the thread contexts to restore the expected value. +*/ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +/* (TCG) Set the OS CAM line of the thread interrupt context. */ +spapr_xive_set_tctx_os_cam (cpu->tctx); + } +return 0 ; }static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp)
Re: [Qemu-devel] [PATCH 06/13] spapr/xive: fix migration of the XiveTCTX under TCG
Quoted 16 lines · Cédric Le Goater
Quoted 10 lines · David Gibson
Quoted 9 lines · Cédric Le Goater
When the thread interrupt management state is retrieved from the KVM VCPU, word2 is saved under the QEMU XIVE thread context to print out the OS CAM line under the QEMU monitor.
This breaks the migration of a TCG guest (and with KVM when kernel_irqchip=off) because the matching algorithm of the presenter relies on the OS CAM value. Fix with an extra reset of the thread contexts to restore the expected value.
Quoted 4 lines
Quoted 1 line
Signed-off-by Cédric Le Goater <[email protected]>Why is the CAM value you get from KVM different from the one you expect in qemu?
Quoted 8 lines
An NVT base identifier is allocated for each VM at the OPAL level and each vCPU getd an increment of this value. It is pushed in the OS CAM line when the vCPU is scheduled to run.
KVM XIVE names this identifier a VP id.
The QEMU emulation of XIVE uses a fixed value for the NVT base identifier.
Ah, I see.
Hmm. Really this highlights why I'm uneasy migrating the whole TCTX as just a blob of registers, even though only some of them are really runtime state, and others are machine configuration that can be worked out separately at the two ends.
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 06/13] spapr/xive: fix migration of the XiveTCTX under TCG
Quoted 26 lines · David Gibson
Quoted 16 lines · Cédric Le Goater
Quoted 10 lines · David Gibson
Quoted 9 lines · Cédric Le Goater
When the thread interrupt management state is retrieved from the KVM VCPU, word2 is saved under the QEMU XIVE thread context to print out the OS CAM line under the QEMU monitor.
This breaks the migration of a TCG guest (and with KVM when kernel_irqchip=off) because the matching algorithm of the presenter relies on the OS CAM value. Fix with an extra reset of the thread contexts to restore the expected value.
Quoted 4 lines
Quoted 1 line
Signed-off-by Cédric Le Goater <[email protected]>Why is the CAM value you get from KVM different from the one you expect in qemu?
Quoted 8 lines
An NVT base identifier is allocated for each VM at the OPAL level and each vCPU getd an increment of this value. It is pushed in the OS CAM line when the vCPU is scheduled to run.
KVM XIVE names this identifier a VP id.
The QEMU emulation of XIVE uses a fixed value for the NVT base identifier.
Quoted 6 lines
Ah, I see.
Hmm. Really this highlights why I'm uneasy migrating the whole TCTX as just a blob of registers, even though only some of them are really runtime state, and others are machine configuration that can be worked out separately at the two ends.
This is really a special case :
1. migration kernel_irqchip=on -> kernel_irqchip=off
2. debug info from KVM is squashing TCG state
We could ignore the VP id configured by KVM but it seems interesting to retrieve.
C.
PATCH 7/13 ppc/xics: introduce a icp_kvm_connect() routine
In reply to Cédric Le GoaterThis routine gathers all the KVM initialization of the XICS KVM presenter. It will be useful when the initialization of the KVM XICS device is moved to a global routine.
--- hw/intc/xics_kvm.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-)
hw/intc/xics_kvm.c +20 -9
@@ -123,11 +123,8 @@ static void icp_kvm_reset(DeviceState *dev)icp_set_kvm_state (ICP (dev),1 ); } -static void icp_kvm_realize (DeviceState *dev,Error **errp) +static void icp_kvm_connect (ICPState *icp,Error **errp) { -ICPState *icp =ICP (dev); -ICPStateClass *icpc =ICP_GET_CLASS (icp); -Error *local_err =NULL ;CPUState *cs;KVMEnabledICP *enabled_icp;unsigned long vcpu_id;
@@ -135,11 +132,6 @@ static void icp_kvm_realize(DeviceState *dev, Error **errp)if (kernel_xics_fd ==-1 ) {abort (); - } - - icpc->parent_realize (dev, &local_err); -if (local_err) { -error_propagate (errp, local_err);return ; }
@@ -168,6 +160,25 @@ static void icp_kvm_realize(DeviceState *dev, Error **errp)QLIST_INSERT_HEAD (&kvm_enabled_icps, enabled_icp, node); } +static void icp_kvm_realize (DeviceState *dev,Error **errp) +{ +ICPStateClass *icpc =ICP_GET_CLASS (dev); +Error *local_err =NULL ; + + icpc->parent_realize (dev, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + +/* Connect the presenter to the VCPU (required for CPU hotplug) */ +icp_kvm_connect (ICP (dev), &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } +} +static void icp_kvm_class_init (ObjectClass *klass,void *data) {DeviceClass *dc =DEVICE_CLASS (klass);
Signature · 1 line
2.20.1
PATCH 8/13 spapr/rtas: modify spapr_rtas_register() to remove RTAS handlers
In reply to Cédric Le GoaterRemoving RTAS handlers will become necessary when the new pseries machine supporting multiple interrupt mode is introduced.
--- include/hw/ppc/spapr.h | 4 ++++ hw/ppc/spapr_rtas.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-)
include/hw/ppc/spapr.h +4 -0
@@ -657,6 +657,10 @@ typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm, uint32_t nargs, target_ulong args, uint32_t nret, target_ulong rets);void spapr_rtas_register (int token,const char *name,spapr_rtas_fn fn); +static inline void spapr_rtas_unregister (int token) +{ +spapr_rtas_register (token,NULL ,NULL ); +}target_ulong spapr_rtas_call (PowerPCCPU *cpu,sPAPRMachineState *sm,uint32_t token,uint32_t nargs,target_ulong args,uint32_t nret,target_ulong rets);
hw/ppc/spapr_rtas.c +1 -1
@@ -404,7 +404,7 @@ void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn) token -=RTAS_TOKEN_BASE ; -assert (!rtas_table[token].name); +assert (!name || !rtas_table[token].name); rtas_table[token].name = name; rtas_table[token].fn = fn;
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 08/13] spapr/rtas: modify spapr_rtas_register() to remove RTAS handlers
Quoted 14 lines · Cédric Le Goater
Removing RTAS handlers will become necessary when the new pseries machine supporting multiple interrupt mode is introduced.
Signed-off-by Cédric Le Goater <[email protected]>--- include/hw/ppc/spapr.h | 4 ++++ hw/ppc/spapr_rtas.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-)include/hw/ppc/spapr.h +0
-0
include/hw/ppc/spapr.h typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm, · line 663
Quoted 7 lines
include/hw/ppc/spapr.h +4
-0@@ -657,6 +657,10 @@ typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm, uint32_t nargs, target_ulong args, uint32_t nret, target_ulong rets);void spapr_rtas_register (int token,const char *name,spapr_rtas_fn fn); +static inline void spapr_rtas_unregister (int token) +{ +spapr_rtas_register (token,NULL ,NULL ); +}
The new helper is not used anywhere.
hw/ppc/spapr_rtas.c void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn) · line 407
Quoted 13 lines
include/hw/ppc/spapr.h +0
-0target_ulong spapr_rtas_call (PowerPCCPU *cpu,sPAPRMachineState *sm,uint32_t token,uint32_t nargs,target_ulong args,uint32_t nret,target_ulong rets);hw/ppc/spapr_rtas.c +1
-1@@ -404,7 +404,7 @@ void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn) token -=RTAS_TOKEN_BASE ; -assert (!rtas_table[token].name); +assert (!name || !rtas_table[token].name);
but allowing name==NULL is.
Quoted 4 lines
hw/ppc/spapr_rtas.c +0
-0rtas_table[token].name = name; rtas_table[token].fn = fn;
Signature · 1 line
Alexey
Re: [Qemu-devel] [PATCH 08/13] spapr/rtas: modify spapr_rtas_register() to remove RTAS handlers
Quoted 12 lines · Alexey Kardashevskiy
Quoted 9 lines · Cédric Le Goater
Removing RTAS handlers will become necessary when the new pseries machine supporting multiple interrupt mode is introduced.
Signed-off-by Cédric Le Goater <[email protected]>--- include/hw/ppc/spapr.h | 4 ++++ hw/ppc/spapr_rtas.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-)
Quoted 14 lines
include/hw/ppc/spapr.h typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm, · line 663
Quoted 12 lines
include/hw/ppc/spapr.h +4
-0@@ -657,6 +657,10 @@ typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm, uint32_t nargs, target_ulong args, uint32_t nret, target_ulong rets);void spapr_rtas_register (int token,const char *name,spapr_rtas_fn fn); +static inline void spapr_rtas_unregister (int token) +{ +spapr_rtas_register (token,NULL ,NULL ); +}The new helper is not used anywhere.
ah yes. patch 10 should be introducing the helper in xics_kvm_disconnect()
I am fixing that.
Thanks,
C.
Quoted 23 lines
hw/ppc/spapr_rtas.c void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn) · line 407
Quoted 13 lines
include/hw/ppc/spapr.h +0
-0target_ulong spapr_rtas_call (PowerPCCPU *cpu,sPAPRMachineState *sm,uint32_t token,uint32_t nargs,target_ulong args,uint32_t nret,target_ulong rets);hw/ppc/spapr_rtas.c +1
-1@@ -404,7 +404,7 @@ void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn) token -=RTAS_TOKEN_BASE ; -assert (!rtas_table[token].name); +assert (!name || !rtas_table[token].name);but allowing name==NULL is.
Quoted 4 lines
hw/ppc/spapr_rtas.c +0
-0rtas_table[token].name = name; rtas_table[token].fn = fn;
PATCH 9/13 sysbus: add a sysbus_mmio_unmap() helper
In reply to Cédric Le GoaterThis will be used to remove the MMIO regions of the POWER9 XIVE interrupt controller when the sPAPR machine is reseted.
--- include/hw/sysbus.h | 1 + hw/core/sysbus.c | 10 ++++++++++ 2 files changed, 11 insertions(+)
include/hw/sysbus.h +1 -0
@@ -89,6 +89,7 @@ qemu_irq sysbus_get_connected_irq(SysBusDevice *dev, int n);void sysbus_mmio_map (SysBusDevice *dev,int n,hwaddr addr);void sysbus_mmio_map_overlap (SysBusDevice *dev,int n,hwaddr addr,int priority); +void sysbus_mmio_unmap (SysBusDevice *dev,int n);void sysbus_add_io (SysBusDevice *dev,hwaddr addr,MemoryRegion *mem);MemoryRegion *sysbus_address_space (SysBusDevice *dev);
hw/core/sysbus.c +10 -0
@@ -153,6 +153,16 @@ static void sysbus_mmio_map_common(SysBusDevice *dev, int n, hwaddr addr, } } +void sysbus_mmio_unmap (SysBusDevice *dev,int n) +{ +assert (n >=0 && n < dev->num_mmio); + +if (dev->mmio[n].addr != (hwaddr)-1 ) { +memory_region_del_subregion (get_system_memory (), dev->mmio[n].memory); + dev->mmio[n].addr = (hwaddr)-1 ; + } +} +void sysbus_mmio_map (SysBusDevice *dev,int n,hwaddr addr) {sysbus_mmio_map_common (dev, n, addr, false,0 );
Signature · 1 line
2.20.1
PATCH 10/13 spapr: introduce routines to delete the KVM IRQ device
In reply to Cédric Le GoaterIf a new interrupt mode is chosen by CAS, the machine generates a reset to reconfigure. At this point, the connection with the previous KVM device needs to be closed and a new connection needs to opened with the KVM device operating the chosen interrupt mode.
New routines are introduced to destroy the XICS and the XIVE KVM devices. They make use of a new KVM device ioctl which destroys the device and also disconnects the IRQ presenters from the vCPUs.
--- include/hw/ppc/spapr_xive.h | 1 + include/hw/ppc/xics.h | 1 + hw/intc/spapr_xive_kvm.c | 60 +++++++++++++++++++++++++++++++++++++ hw/intc/xics_kvm.c | 57 +++++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+)
include/hw/ppc/spapr_xive.h +1 -0
@@ -70,6 +70,7 @@ void spapr_xive_map_mmio(sPAPRXive *xive);* KVM XIVE device helpers */ void kvmppc_xive_connect (sPAPRXive *xive,Error **errp); +void kvmppc_xive_disconnect (sPAPRXive *xive,Error **errp);void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp);int kvmppc_xive_pre_save (sPAPRXive *xive);int kvmppc_xive_post_load (sPAPRXive *xive,int version_id);
include/hw/ppc/xics.h +1 -0
@@ -205,6 +205,7 @@ typedef struct sPAPRMachineState sPAPRMachineState;void spapr_dt_xics (sPAPRMachineState *spapr,uint32_t nr_servers,void *fdt,uint32_t phandle);int xics_kvm_init (sPAPRMachineState *spapr,Error **errp); +int xics_kvm_disconnect (sPAPRMachineState *spapr,Error **errp);void xics_spapr_init (sPAPRMachineState *spapr); Object *icp_create(Object *cpu, const char *type, XICSFabric *xi,
hw/intc/spapr_xive_kvm.c +60 -0
69 lines not on this page · raw message
hw/intc/xics_kvm.c +57 -0
69 lines not on this page · raw message
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 10/13] spapr: introduce routines to delete the KVM IRQ device
Quoted 10 lines · Cédric Le Goater
If a new interrupt mode is chosen by CAS, the machine generates a reset to reconfigure. At this point, the connection with the previous KVM device needs to be closed and a new connection needs to opened with the KVM device operating the chosen interrupt mode.
New routines are introduced to destroy the XICS and the XIVE KVM devices. They make use of a new KVM device ioctl which destroys the device and also disconnects the IRQ presenters from the vCPUs.
Signed-off-by Cédric Le Goater <[email protected]>
Looks reasonable.
Quoted 181 lines
--- include/hw/ppc/spapr_xive.h | 1 + include/hw/ppc/xics.h | 1 + hw/intc/spapr_xive_kvm.c | 60 +++++++++++++++++++++++++++++++++++++ hw/intc/xics_kvm.c | 57 +++++++++++++++++++++++++++++++++++ 4 files changed, 119 insertions(+)include/hw/ppc/spapr_xive.h +1
-0@@ -70,6 +70,7 @@ void spapr_xive_map_mmio(sPAPRXive *xive);* KVM XIVE device helpers */ void kvmppc_xive_connect (sPAPRXive *xive,Error **errp); +void kvmppc_xive_disconnect (sPAPRXive *xive,Error **errp);void kvmppc_xive_synchronize_state (sPAPRXive *xive,Error **errp);int kvmppc_xive_pre_save (sPAPRXive *xive);int kvmppc_xive_post_load (sPAPRXive *xive,int version_id);include/hw/ppc/xics.h +1
-0@@ -205,6 +205,7 @@ typedef struct sPAPRMachineState sPAPRMachineState;void spapr_dt_xics (sPAPRMachineState *spapr,uint32_t nr_servers,void *fdt,uint32_t phandle);int xics_kvm_init (sPAPRMachineState *spapr,Error **errp); +int xics_kvm_disconnect (sPAPRMachineState *spapr,Error **errp);void xics_spapr_init (sPAPRMachineState *spapr); Object *icp_create(Object *cpu, const char *type, XICSFabric *xi,hw/intc/spapr_xive_kvm.c +60
-0@@ -57,6 +57,16 @@ static void kvm_cpu_enable(CPUState *cs)QLIST_INSERT_HEAD (&kvm_enabled_cpus, enabled_cpu, node); } +static void kvm_cpu_disable_all (void ) +{ +KVMEnabledCPU *enabled_cpu, *next; + + QLIST_FOREACH_SAFE(enabled_cpu, &kvm_enabled_cpus, node, next) { +QLIST_REMOVE (enabled_cpu, node); +g_free (enabled_cpu); + } +} +/* * XIVE Thread Interrupt Management context (KVM) */ @@ -769,3 +779,53 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)/* Map all regions */ spapr_xive_map_mmio (xive); } + +void kvmppc_xive_disconnect (sPAPRXive *xive,Error **errp) +{ +XiveSource *xsrc; +struct kvm_create_device xive_destroy_device = {0 }; +size_t esb_len; +int rc; + +if (!kvm_enabled () || !kvmppc_has_cap_xive ()) { +error_setg (errp,"IRQ_XIVE capability must be present for KVM" ); +return ; + } + +/* The KVM XIVE device is not in use */ +if (!xive || xive->fd ==-1 ) { +return ; + } + +/* Clear the KVM mapping */ + xsrc = &xive->source; + esb_len = (1ull << xsrc->esb_shift) * xsrc->nr_irqs; + +sysbus_mmio_unmap (SYS_BUS_DEVICE (xive),0 ); +munmap (xsrc->esb_mmap, esb_len); + +sysbus_mmio_unmap (SYS_BUS_DEVICE (xive),1 ); + +sysbus_mmio_unmap (SYS_BUS_DEVICE (xive),2 ); +munmap (xive->tm_mmap,4ull <<TM_SHIFT ); + +/* Destroy the KVM device. This also clears the VCPU presenters */ + xive_destroy_device.fd = xive->fd; + xive_destroy_device.type =KVM_DEV_TYPE_XIVE ; + rc =kvm_vm_ioctl (kvm_state,KVM_DESTROY_DEVICE , &xive_destroy_device); +if (rc <0 ) { +error_setg_errno (errp, -rc,"Error on KVM_DESTROY_DEVICE for XIVE" ); + } +close (xive->fd); + xive->fd =-1 ; + + kvm_kernel_irqchip = false; + kvm_msi_via_irqfd_allowed = false; + kvm_gsi_direct_mapping = false; + +/* Clear the local list of presenter (hotplug) */ +kvm_cpu_disable_all (); + +/* VM Change state handler is not needed anymore */ +qemu_del_vm_change_state_handler (xive->change); +}hw/intc/xics_kvm.c +57
-0@@ -50,6 +50,16 @@ typedef struct KVMEnabledICP { static QLIST_HEAD(, KVMEnabledICP) kvm_enabled_icps =QLIST_HEAD_INITIALIZER (&kvm_enabled_icps); +static void kvm_disable_icps (void ) +{ +KVMEnabledICP *enabled_icp, *next; + + QLIST_FOREACH_SAFE(enabled_icp, &kvm_enabled_icps, node, next) { +QLIST_REMOVE (enabled_icp, node); +g_free (enabled_icp); + } +} +/* * ICP-KVM */ @@ -455,6 +465,53 @@ fail:return -1 ; } +int xics_kvm_disconnect (sPAPRMachineState *spapr,Error **errp) +{ +int rc; +struct kvm_create_device xics_create_device = { + .fd = kernel_xics_fd, + .type =KVM_DEV_TYPE_XICS , + .flags =0 , + }; + +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return 0 ; + } + +if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { +error_setg (errp, +"KVM and IRQ_XICS capability must be present for KVM XICS device" ); +return -1 ; + } + + rc =kvm_vm_ioctl (kvm_state,KVM_DESTROY_DEVICE , &xics_create_device); +if (rc <0 ) { +error_setg_errno (errp, -rc,"Error on KVM_DESTROY_DEVICE for XICS" ); + } +close (kernel_xics_fd); + kernel_xics_fd =-1 ; + +spapr_rtas_register (RTAS_IBM_SET_XIVE ,NULL ,0 ); +spapr_rtas_register (RTAS_IBM_GET_XIVE ,NULL ,0 ); +spapr_rtas_register (RTAS_IBM_INT_OFF ,NULL ,0 ); +spapr_rtas_register (RTAS_IBM_INT_ON ,NULL ,0 ); + +kvmppc_define_rtas_kernel_token (0 ,"ibm,set-xive" ); +kvmppc_define_rtas_kernel_token (0 ,"ibm,get-xive" ); +kvmppc_define_rtas_kernel_token (0 ,"ibm,int-on" ); +kvmppc_define_rtas_kernel_token (0 ,"ibm,int-off" ); + + kvm_kernel_irqchip = false; + kvm_msi_via_irqfd_allowed = false; + kvm_gsi_direct_mapping = false; + +/* Clear the presenter from the VCPUs */ +kvm_disable_icps (); + +return rc; +} +static void xics_kvm_register_types (void ) {type_register_static (&ics_kvm_info);
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 11/13 spapr: check for the activation of the KVM IRQ device
In reply to Cédric Le GoaterThe activation of the KVM IRQ device depends on the interrupt mode chosen at CAS time by the machine and some methods used at reset or by the migration need to be protected.
--- hw/intc/spapr_xive_kvm.c | 28 ++++++++++++++++++++++++++++ hw/intc/xics_kvm.c | 25 ++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-)
hw/intc/spapr_xive_kvm.c +28 -0
@@ -95,9 +95,15 @@ static void kvmppc_xive_cpu_set_state(XiveTCTX *tctx, Error **errp)void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp) { +sPAPRXive *xive =SPAPR_MACHINE (qdev_get_machine ())->xive;uint64_t state[4 ] = {0 };int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } + ret =kvm_get_one_reg (tctx->cs,KVM_REG_PPC_NVT_STATE , state);if (ret !=0 ) { error_setg_errno(errp, errno,
@@ -151,6 +157,11 @@ void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)unsigned long vcpu_id;int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } +/* Check if CPU was hot unplugged and replugged. */ if (kvm_cpu_is_enabled (tctx->cs)) {return ;
@@ -234,9 +245,13 @@ static void kvmppc_xive_source_get_state(XiveSource *xsrc)void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val) {XiveSource *xsrc = opaque; +sPAPRXive *xive =SPAPR_XIVE (xsrc->xive);struct kvm_irq_level args;int rc; +/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 ); + args.irq = srcno;if (!xive_source_irq_is_lsi (xsrc, srcno)) {if (!val) {
@@ -580,6 +595,11 @@ int kvmppc_xive_pre_save(sPAPRXive *xive)Error *local_err =NULL ;CPUState *cs; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return 0 ; + } +/* Grab the EAT */ kvmppc_xive_get_eas_state (xive, &local_err);if (local_err) {
@@ -612,6 +632,9 @@ int kvmppc_xive_post_load(sPAPRXive *xive, int version_id)Error *local_err =NULL ;CPUState *cs; +/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 ); +/* Restore the ENDT first. The targetting depends on it. */ CPU_FOREACH (cs) {kvmppc_xive_set_eq_state (xive, cs, &local_err);
@@ -649,6 +672,11 @@ void kvmppc_xive_synchronize_state(sPAPRXive *xive, Error **errp)CPUState *cs;Error *local_err =NULL ; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } +/* * When the VM is stopped, the sources are masked and the previous * state is saved in anticipation of a migration. We should not
hw/intc/xics_kvm.c +24 -1
@@ -68,6 +68,11 @@ static void icp_get_kvm_state(ICPState *icp)uint64_t state;int ret; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return ; + } +/* ICP for this CPU thread is not in use, exiting */ if (!icp->cs) {return ;
@@ -104,6 +109,11 @@ static int icp_set_kvm_state(ICPState *icp, int version_id)uint64_t state;int ret; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return 0 ; + } +/* ICP for this CPU thread is not in use, exiting */ if (!icp->cs) {return 0 ;
@@ -140,8 +150,8 @@ static void icp_kvm_connect(ICPState *icp, Error **errp)unsigned long vcpu_id;int ret; +/* The KVM XICS device is not in use */ if (kernel_xics_fd ==-1 ) { -abort ();return ; }
@@ -220,6 +230,11 @@ static void ics_get_kvm_state(ICSState *ics)uint64_t state;int i; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return ; + } +for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];
@@ -279,6 +294,11 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)int i;Error *local_err =NULL ; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return 0 ; + } +for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];int ret;
@@ -325,6 +345,9 @@ void ics_kvm_set_irq(void *opaque, int srcno, int val)struct kvm_irq_level args;int rc; +/* The KVM XICS device should be in use */ +assert (kernel_xics_fd !=-1 ); + args.irq = srcno + ics->offset;if (ics->irqs[srcno].flags &XICS_FLAGS_IRQ_MSI ) {if (!val) {
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 11/13] spapr: check for the activation of the KVM IRQ device
Quoted 69 lines · Cédric Le Goater
The activation of the KVM IRQ device depends on the interrupt mode chosen at CAS time by the machine and some methods used at reset or by the migration need to be protected.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive_kvm.c | 28 ++++++++++++++++++++++++++++ hw/intc/xics_kvm.c | 25 ++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-)hw/intc/spapr_xive_kvm.c +20
-0@@ -95,9 +95,15 @@ static void kvmppc_xive_cpu_set_state(XiveTCTX *tctx, Error **errp)void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp) { +sPAPRXive *xive =SPAPR_MACHINE (qdev_get_machine ())->xive;uint64_t state[4 ] = {0 };int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } + ret =kvm_get_one_reg (tctx->cs,KVM_REG_PPC_NVT_STATE , state);if (ret !=0 ) { error_setg_errno(errp, errno,@@ -151,6 +157,11 @@ void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)unsigned long vcpu_id;int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } +/* Check if CPU was hot unplugged and replugged. */ if (kvm_cpu_is_enabled (tctx->cs)) {return ;@@ -234,9 +245,13 @@ static void kvmppc_xive_source_get_state(XiveSource *xsrc)void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val) {XiveSource *xsrc = opaque; +sPAPRXive *xive =SPAPR_XIVE (xsrc->xive);struct kvm_irq_level args;int rc; +/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 ); + args.irq = srcno;if (!xive_source_irq_is_lsi (xsrc, srcno)) {if (!val) {@@ -580,6 +595,11 @@ int kvmppc_xive_pre_save(sPAPRXive *xive)Error *local_err =NULL ;CPUState *cs; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return 0 ; + } +/* Grab the EAT */ kvmppc_xive_get_eas_state (xive, &local_err);if (local_err) {
hw/intc/spapr_xive_kvm.c int kvmppc_xive_post_load(sPAPRXive *xive, int version_id) · line 636
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +2
-0@@ -612,6 +632,9 @@ int kvmppc_xive_post_load(sPAPRXive *xive, int version_id)Error *local_err =NULL ;CPUState *cs; +/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 );
I'm guessing this is an assert() because the handler shouldn't be registered when we're not in KVM mode. But wouldn't that also be true of the pre_save hook, which errors out rather than asserting?
Quoted 87 lines
hw/intc/spapr_xive_kvm.c +5
-0/* Restore the ENDT first. The targetting depends on it. */ CPU_FOREACH (cs) {kvmppc_xive_set_eq_state (xive, cs, &local_err);@@ -649,6 +672,11 @@ void kvmppc_xive_synchronize_state(sPAPRXive *xive, Error **errp)CPUState *cs;Error *local_err =NULL ; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } +/* * When the VM is stopped, the sources are masked and the previous * state is saved in anticipation of a migration. We should not hw/intc/xics_kvm.c +24
-1@@ -68,6 +68,11 @@ static void icp_get_kvm_state(ICPState *icp)uint64_t state;int ret; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return ; + } +/* ICP for this CPU thread is not in use, exiting */ if (!icp->cs) {return ;@@ -104,6 +109,11 @@ static int icp_set_kvm_state(ICPState *icp, int version_id)uint64_t state;int ret; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return 0 ; + } +/* ICP for this CPU thread is not in use, exiting */ if (!icp->cs) {return 0 ;@@ -140,8 +150,8 @@ static void icp_kvm_connect(ICPState *icp, Error **errp)unsigned long vcpu_id;int ret; +/* The KVM XICS device is not in use */ if (kernel_xics_fd ==-1 ) { -abort ();return ; }@@ -220,6 +230,11 @@ static void ics_get_kvm_state(ICSState *ics)uint64_t state;int i; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return ; + } +for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];@@ -279,6 +294,11 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)int i;Error *local_err =NULL ; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return 0 ; + } +for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];int ret;@@ -325,6 +345,9 @@ void ics_kvm_set_irq(void *opaque, int srcno, int val)struct kvm_irq_level args;int rc; +/* The KVM XICS device should be in use */ +assert (kernel_xics_fd !=-1 ); + args.irq = srcno + ics->offset;if (ics->irqs[srcno].flags &XICS_FLAGS_IRQ_MSI ) {if (!val) {
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 11/13] spapr: check for the activation of the KVM IRQ device
Quoted 73 lines · David Gibson
Quoted 72 lines · Cédric Le Goater
The activation of the KVM IRQ device depends on the interrupt mode chosen at CAS time by the machine and some methods used at reset or by the migration need to be protected.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive_kvm.c | 28 ++++++++++++++++++++++++++++ hw/intc/xics_kvm.c | 25 ++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-)hw/intc/spapr_xive_kvm.c +20
-0@@ -95,9 +95,15 @@ static void kvmppc_xive_cpu_set_state(XiveTCTX *tctx, Error **errp)void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp) { +sPAPRXive *xive =SPAPR_MACHINE (qdev_get_machine ())->xive;uint64_t state[4 ] = {0 };int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } + ret =kvm_get_one_reg (tctx->cs,KVM_REG_PPC_NVT_STATE , state);if (ret !=0 ) { error_setg_errno(errp, errno,@@ -151,6 +157,11 @@ void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)unsigned long vcpu_id;int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } +/* Check if CPU was hot unplugged and replugged. */ if (kvm_cpu_is_enabled (tctx->cs)) {return ;@@ -234,9 +245,13 @@ static void kvmppc_xive_source_get_state(XiveSource *xsrc)void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val) {XiveSource *xsrc = opaque; +sPAPRXive *xive =SPAPR_XIVE (xsrc->xive);struct kvm_irq_level args;int rc; +/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 ); + args.irq = srcno;if (!xive_source_irq_is_lsi (xsrc, srcno)) {if (!val) {@@ -580,6 +595,11 @@ int kvmppc_xive_pre_save(sPAPRXive *xive)Error *local_err =NULL ;CPUState *cs; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return 0 ; + } +/* Grab the EAT */ kvmppc_xive_get_eas_state (xive, &local_err);if (local_err) {@@ -612,6 +632,9 @@ int kvmppc_xive_post_load(sPAPRXive *xive, int version_id)Error *local_err =NULL ;CPUState *cs;
Quoted 6 lines
hw/intc/spapr_xive_kvm.c int kvmppc_xive_post_load(sPAPRXive *xive, int version_id) · line 636
Quoted 2 lines
hw/intc/spapr_xive_kvm.c +2
-0+/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 );I'm guessing this is an
assert()because the handler shouldn't be registered when we're not in KVM mode. But wouldn't that also be true of thepre_savehook, which errors out rather than asserting?
The handlers are not symetric.
The pre_save is registered in the vmstate of the sPAPRXive model and the post_load is handled at the machine level after all XIVE state have been transferred.
C.
Quoted 89 lines
Quoted 88 lines
hw/intc/spapr_xive_kvm.c +5
-0/* Restore the ENDT first. The targetting depends on it. */ CPU_FOREACH (cs) {kvmppc_xive_set_eq_state (xive, cs, &local_err);@@ -649,6 +672,11 @@ void kvmppc_xive_synchronize_state(sPAPRXive *xive, Error **errp)CPUState *cs;Error *local_err =NULL ; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } +/* * When the VM is stopped, the sources are masked and the previous * state is saved in anticipation of a migration. We should not hw/intc/xics_kvm.c +24
-1@@ -68,6 +68,11 @@ static void icp_get_kvm_state(ICPState *icp)uint64_t state;int ret; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return ; + } +/* ICP for this CPU thread is not in use, exiting */ if (!icp->cs) {return ;@@ -104,6 +109,11 @@ static int icp_set_kvm_state(ICPState *icp, int version_id)uint64_t state;int ret; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return 0 ; + } +/* ICP for this CPU thread is not in use, exiting */ if (!icp->cs) {return 0 ;@@ -140,8 +150,8 @@ static void icp_kvm_connect(ICPState *icp, Error **errp)unsigned long vcpu_id;int ret; +/* The KVM XICS device is not in use */ if (kernel_xics_fd ==-1 ) { -abort ();return ; }@@ -220,6 +230,11 @@ static void ics_get_kvm_state(ICSState *ics)uint64_t state;int i; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return ; + } +for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];@@ -279,6 +294,11 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)int i;Error *local_err =NULL ; +/* The KVM XICS device is not in use */ +if (kernel_xics_fd ==-1 ) { +return 0 ; + } +for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];int ret;@@ -325,6 +345,9 @@ void ics_kvm_set_irq(void *opaque, int srcno, int val)struct kvm_irq_level args;int rc; +/* The KVM XICS device should be in use */ +assert (kernel_xics_fd !=-1 ); + args.irq = srcno + ics->offset;if (ics->irqs[srcno].flags &XICS_FLAGS_IRQ_MSI ) {if (!val) {
Re: [Qemu-devel] [PATCH 11/13] spapr: check for the activation of the KVM IRQ device
Quoted 81 lines · Cédric Le Goater
Quoted 73 lines · David Gibson
Quoted 72 lines · Cédric Le Goater
The activation of the KVM IRQ device depends on the interrupt mode chosen at CAS time by the machine and some methods used at reset or by the migration need to be protected.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive_kvm.c | 28 ++++++++++++++++++++++++++++ hw/intc/xics_kvm.c | 25 ++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-)hw/intc/spapr_xive_kvm.c +20
-0@@ -95,9 +95,15 @@ static void kvmppc_xive_cpu_set_state(XiveTCTX *tctx, Error **errp)void kvmppc_xive_cpu_get_state (XiveTCTX *tctx,Error **errp) { +sPAPRXive *xive =SPAPR_MACHINE (qdev_get_machine ())->xive;uint64_t state[4 ] = {0 };int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } + ret =kvm_get_one_reg (tctx->cs,KVM_REG_PPC_NVT_STATE , state);if (ret !=0 ) { error_setg_errno(errp, errno,@@ -151,6 +157,11 @@ void kvmppc_xive_cpu_connect(XiveTCTX *tctx, Error **errp)unsigned long vcpu_id;int ret; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return ; + } +/* Check if CPU was hot unplugged and replugged. */ if (kvm_cpu_is_enabled (tctx->cs)) {return ;@@ -234,9 +245,13 @@ static void kvmppc_xive_source_get_state(XiveSource *xsrc)void kvmppc_xive_source_set_irq (void *opaque,int srcno,int val) {XiveSource *xsrc = opaque; +sPAPRXive *xive =SPAPR_XIVE (xsrc->xive);struct kvm_irq_level args;int rc; +/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 ); + args.irq = srcno;if (!xive_source_irq_is_lsi (xsrc, srcno)) {if (!val) {@@ -580,6 +595,11 @@ int kvmppc_xive_pre_save(sPAPRXive *xive)Error *local_err =NULL ;CPUState *cs; +/* The KVM XIVE device is not in use */ +if (xive->fd ==-1 ) { +return 0 ; + } +/* Grab the EAT */ kvmppc_xive_get_eas_state (xive, &local_err);if (local_err) {@@ -612,6 +632,9 @@ int kvmppc_xive_post_load(sPAPRXive *xive, int version_id)Error *local_err =NULL ;CPUState *cs;Quoted 6 lines
hw/intc/spapr_xive_kvm.c int kvmppc_xive_post_load(sPAPRXive *xive, int version_id) · line 636
Quoted 2 lines
hw/intc/spapr_xive_kvm.c +2
-0+/* The KVM XIVE device should be in use */ +assert (xive->fd !=-1 );I'm guessing this is an
assert()because the handler shouldn't be registered when we're not in KVM mode. But wouldn't that also be true of thepre_savehook, which errors out rather than asserting?
Quoted 5 lines
The handlers are not symetric.
The
pre_saveis registered in the vmstate of the sPAPRXive model and thepost_loadis handled at the machine level after all XIVE state have been transferred.
Ah, ok. Some comments on site explaining why that's so would be useful.
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
PATCH 12/13 spapr/xics: ignore the lower 4K in the IRQ number space
In reply to Cédric Le GoaterThe IRQ number space of the XIVE and XICS interrupt mode are aligned when using the dual interrupt mode for the machine. This means that the ICS offset is set to zero in QEMU and that the KVM XICS device should be informed of this new value. Unfortunately, there is now way to do so and KVM still maintains the XICS_IRQ_BASE (0x1000) offset.
Ignore the lower 4K which are not used under the XICS interrupt mode. These IRQ numbers are only claimed by XIVE for the CPU IPIs.
--- hw/intc/xics_kvm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
hw/intc/xics_kvm.c +18 -0
@@ -238,6 +238,15 @@ static void ics_get_kvm_state(ICSState *ics)for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i]; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } +kvm_device_access (kernel_xics_fd,KVM_DEV_XICS_GRP_SOURCES , i + ics->offset, &state, false, &error_fatal);
@@ -303,6 +312,15 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)ICSIRQState *irq = &ics->irqs[i];int ret; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } + state = irq->server; state |= (uint64_t )(irq->saved_priority &KVM_XICS_PRIORITY_MASK ) <<KVM_XICS_PRIORITY_SHIFT ;
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 12/13] spapr/xics: ignore the lower 4K in the IRQ number space
Quoted 21 lines · Cédric Le Goater
The IRQ number space of the XIVE and XICS interrupt mode are aligned when using the dual interrupt mode for the machine. This means that the ICS offset is set to zero in QEMU and that the KVM XICS device should be informed of this new value. Unfortunately, there is now way to do so and KVM still maintains the
XICS_IRQ_BASE(0x1000) offset.Ignore the lower 4K which are not used under the XICS interrupt mode. These IRQ numbers are only claimed by XIVE for the CPU IPIs.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/xics_kvm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)hw/intc/xics_kvm.c +0
-0@@ -238,6 +238,15 @@ static void ics_get_kvm_state(ICSState *ics)for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];
hw/intc/xics_kvm.c static void ics_get_kvm_state(ICSState *ics) · line 249
Quoted 10 lines
hw/intc/xics_kvm.c +9
-0+/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } +
This seems bogus to me. The guest-visible irq numbers need to line up between xics and xive mode, yes, but that doesn't mean we need to keep around a great big array of unused array of ICS irq states, even in TCG mode.
Quoted 19 lines
hw/intc/xics_kvm.c +9
-0kvm_device_access (kernel_xics_fd,KVM_DEV_XICS_GRP_SOURCES , i + ics->offset, &state, false, &error_fatal);@@ -303,6 +312,15 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)ICSIRQState *irq = &ics->irqs[i];int ret; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } + state = irq->server; state |= (uint64_t )(irq->saved_priority &KVM_XICS_PRIORITY_MASK ) <<KVM_XICS_PRIORITY_SHIFT ;
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 12/13] spapr/xics: ignore the lower 4K in the IRQ number space
Quoted 23 lines · David Gibson
Quoted 22 lines · Cédric Le Goater
The IRQ number space of the XIVE and XICS interrupt mode are aligned when using the dual interrupt mode for the machine. This means that the ICS offset is set to zero in QEMU and that the KVM XICS device should be informed of this new value. Unfortunately, there is now way to do so and KVM still maintains the
XICS_IRQ_BASE(0x1000) offset.Ignore the lower 4K which are not used under the XICS interrupt mode. These IRQ numbers are only claimed by XIVE for the CPU IPIs.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/xics_kvm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)hw/intc/xics_kvm.c +0
-0@@ -238,6 +238,15 @@ static void ics_get_kvm_state(ICSState *ics)for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];
Quoted 14 lines
hw/intc/xics_kvm.c static void ics_get_kvm_state(ICSState *ics) · line 249
Quoted 9 lines
hw/intc/xics_kvm.c +9
-0+/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } +This seems bogus to me. The guest-visible irq numbers need to line up between xics and xive mode, yes, but that doesn't mean we need to keep around a great big array of unused array of ICS irq states, even in TCG mode.
This is because the qirqs[] array is under the machine and shared between both interrupt modes, xics and xive.
C.
Quoted 21 lines
Quoted 20 lines
hw/intc/xics_kvm.c +9
-0kvm_device_access (kernel_xics_fd,KVM_DEV_XICS_GRP_SOURCES , i + ics->offset, &state, false, &error_fatal);@@ -303,6 +312,15 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)ICSIRQState *irq = &ics->irqs[i];int ret; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } + state = irq->server; state |= (uint64_t )(irq->saved_priority &KVM_XICS_PRIORITY_MASK ) <<KVM_XICS_PRIORITY_SHIFT ;
Re: [Qemu-devel] [PATCH 12/13] spapr/xics: ignore the lower 4K in the IRQ number space
Quoted 39 lines · Cédric Le Goater
Quoted 23 lines · David Gibson
Quoted 22 lines · Cédric Le Goater
The IRQ number space of the XIVE and XICS interrupt mode are aligned when using the dual interrupt mode for the machine. This means that the ICS offset is set to zero in QEMU and that the KVM XICS device should be informed of this new value. Unfortunately, there is now way to do so and KVM still maintains the
XICS_IRQ_BASE(0x1000) offset.Ignore the lower 4K which are not used under the XICS interrupt mode. These IRQ numbers are only claimed by XIVE for the CPU IPIs.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/xics_kvm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)hw/intc/xics_kvm.c +0
-0@@ -238,6 +238,15 @@ static void ics_get_kvm_state(ICSState *ics)for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];Quoted 14 lines
hw/intc/xics_kvm.c static void ics_get_kvm_state(ICSState *ics) · line 249
Quoted 9 lines
hw/intc/xics_kvm.c +9
-0+/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } +This seems bogus to me. The guest-visible irq numbers need to line up between xics and xive mode, yes, but that doesn't mean we need to keep around a great big array of unused array of ICS irq states, even in TCG mode.
Quoted 2 lines
This is because the qirqs[] array is under the machine and shared between both interrupt modes, xics and xive.
I don't see how that follows. ICSIRQState is indexed in terms of the ICS source number, not the global irq number, so I don't see why it has to match up with the qirq array.
Quoted 25 lines
C.
Quoted 21 lines
Quoted 20 lines
hw/intc/xics_kvm.c +9
-0kvm_device_access (kernel_xics_fd,KVM_DEV_XICS_GRP_SOURCES , i + ics->offset, &state, false, &error_fatal);@@ -303,6 +312,15 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)ICSIRQState *irq = &ics->irqs[i];int ret; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } + state = irq->server; state |= (uint64_t )(irq->saved_priority &KVM_XICS_PRIORITY_MASK ) <<KVM_XICS_PRIORITY_SHIFT ;
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 12/13] spapr/xics: ignore the lower 4K in the IRQ number space
Quoted 40 lines · David Gibson
Quoted 39 lines · Cédric Le Goater
Quoted 37 lines · David Gibson
Quoted 21 lines · Cédric Le Goater
The IRQ number space of the XIVE and XICS interrupt mode are aligned when using the dual interrupt mode for the machine. This means that the ICS offset is set to zero in QEMU and that the KVM XICS device should be informed of this new value. Unfortunately, there is now way to do so and KVM still maintains the
XICS_IRQ_BASE(0x1000) offset.Ignore the lower 4K which are not used under the XICS interrupt mode. These IRQ numbers are only claimed by XIVE for the CPU IPIs.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/xics_kvm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)hw/intc/xics_kvm.c +0
-0@@ -238,6 +238,15 @@ static void ics_get_kvm_state(ICSState *ics)for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];hw/intc/xics_kvm.c static void ics_get_kvm_state(ICSState *ics) · line 249
Quoted 10 lines
hw/intc/xics_kvm.c +9
-0+/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } +This seems bogus to me. The guest-visible irq numbers need to line up between xics and xive mode, yes, but that doesn't mean we need to keep around a great big array of unused array of ICS irq states, even in TCG mode.
Quoted 6 lines
Quoted 2 lines
This is because the qirqs[] array is under the machine and shared between both interrupt modes, xics and xive.
I don't see how that follows. ICSIRQState is indexed in terms of the ICS source number, not the global irq number, so I don't see why it has to match up with the qirq array.
The root cause is the use of spapr->irq->nr_irqs to initialize the ICS and sPAPRXive object. In case of the 'dual' backend, it covers the full XIVE IRQ number space (0x2000 today) but XICS only needs 0x1000.
I think we can fix the offset issue by using the appropriate nr_irqs which should be for the XICS backend : spapr->irq->nr_irqs - ics->offset
I keep in mind the XIVE support for nested guests and I think we will need to extend the IRQ number space in L1 and have the L2 use a portion of it (using an offset).
C.
Quoted 23 lines
Quoted 22 lines
Quoted 21 lines
Quoted 20 lines
hw/intc/xics_kvm.c +9
-0kvm_device_access (kernel_xics_fd,KVM_DEV_XICS_GRP_SOURCES , i + ics->offset, &state, false, &error_fatal);@@ -303,6 +312,15 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)ICSIRQState *irq = &ics->irqs[i];int ret; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } + state = irq->server; state |= (uint64_t )(irq->saved_priority &KVM_XICS_PRIORITY_MASK ) <<KVM_XICS_PRIORITY_SHIFT ;
Re: [Qemu-devel] [Qemu-ppc] [PATCH 12/13] spapr/xics: ignore the lower 4K in the IRQ number space
Quoted 48 lines · On Wed, 13 Feb 2019 09:03:33 +0100 Cédric Le Goater <[email protected]> wrote:
Quoted 40 lines · David Gibson
Quoted 39 lines · Cédric Le Goater
Quoted 37 lines · David Gibson
Quoted 21 lines · Cédric Le Goater
The IRQ number space of the XIVE and XICS interrupt mode are aligned when using the dual interrupt mode for the machine. This means that the ICS offset is set to zero in QEMU and that the KVM XICS device should be informed of this new value. Unfortunately, there is now way to do so and KVM still maintains the
XICS_IRQ_BASE(0x1000) offset.Ignore the lower 4K which are not used under the XICS interrupt mode. These IRQ numbers are only claimed by XIVE for the CPU IPIs.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/xics_kvm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)hw/intc/xics_kvm.c +0
-0@@ -238,6 +238,15 @@ static void ics_get_kvm_state(ICSState *ics)for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];hw/intc/xics_kvm.c static void ics_get_kvm_state(ICSState *ics) · line 249
Quoted 10 lines
hw/intc/xics_kvm.c +9
-0+/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } +This seems bogus to me. The guest-visible irq numbers need to line up between xics and xive mode, yes, but that doesn't mean we need to keep around a great big array of unused array of ICS irq states, even in TCG mode.
Quoted 6 lines
Quoted 2 lines
This is because the qirqs[] array is under the machine and shared between both interrupt modes, xics and xive.
I don't see how that follows. ICSIRQState is indexed in terms of the ICS source number, not the global irq number, so I don't see why it has to match up with the qirq array.
Quoted 7 lines
The root cause is the use of
spapr->irq->nr_irqsto initialize the ICS and sPAPRXive object. In case of the 'dual' backend, it covers the full XIVE IRQ number space (0x2000today) but XICS only needs0x1000.I think we can fix the offset issue by using the appropriate
nr_irqswhich should be for the XICS backend :spapr->irq->nr_irqs-ics->offset
Since the root cause is that the value of spapr->irq->nr_irqs should be different in XIVE and XICS, what about fixing it during reset ?
Something like:
static void spapr_irq_reset_dual(sPAPRMachineState *spapr, Error **errp)
{
[...]
spapr->irq->nr_irqs = spapr_irq_current(spapr)->nr_irqs;
spapr_irq_current(spapr)->reset(spapr, errp);
}Quoted 32 lines
I keep in mind the XIVE support for nested guests and I think we will need to extend the IRQ number space in L1 and have the L2 use a portion of it (using an offset).
C.
Quoted 23 lines
Quoted 22 lines
Quoted 21 lines
Quoted 20 lines
hw/intc/xics_kvm.c +9
-0kvm_device_access (kernel_xics_fd,KVM_DEV_XICS_GRP_SOURCES , i + ics->offset, &state, false, &error_fatal);@@ -303,6 +312,15 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)ICSIRQState *irq = &ics->irqs[i];int ret; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } + state = irq->server; state |= (uint64_t )(irq->saved_priority &KVM_XICS_PRIORITY_MASK ) <<KVM_XICS_PRIORITY_SHIFT ;
Re: [Qemu-devel] [Qemu-ppc] [PATCH 12/13] spapr/xics: ignore the lower 4K in the IRQ number space
Quoted 55 lines · On Wed, 13 Feb 2019 12:27:13 +0100 Greg Kurz <[email protected]> wrote:
Quoted 52 lines · On Wed, 13 Feb 2019 09:03:33 +0100 Cédric Le Goater <[email protected]> wrote:
Quoted 40 lines · David Gibson
Quoted 39 lines · Cédric Le Goater
Quoted 37 lines · David Gibson
Quoted 21 lines · Cédric Le Goater
The IRQ number space of the XIVE and XICS interrupt mode are aligned when using the dual interrupt mode for the machine. This means that the ICS offset is set to zero in QEMU and that the KVM XICS device should be informed of this new value. Unfortunately, there is now way to do so and KVM still maintains the
XICS_IRQ_BASE(0x1000) offset.Ignore the lower 4K which are not used under the XICS interrupt mode. These IRQ numbers are only claimed by XIVE for the CPU IPIs.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/xics_kvm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)hw/intc/xics_kvm.c +0
-0@@ -238,6 +238,15 @@ static void ics_get_kvm_state(ICSState *ics)for (i =0 ; i < ics->nr_irqs; i++) {ICSIRQState *irq = &ics->irqs[i];hw/intc/xics_kvm.c static void ics_get_kvm_state(ICSState *ics) · line 249
Quoted 10 lines
hw/intc/xics_kvm.c +9
-0+/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } +This seems bogus to me. The guest-visible irq numbers need to line up between xics and xive mode, yes, but that doesn't mean we need to keep around a great big array of unused array of ICS irq states, even in TCG mode.
Quoted 6 lines
Quoted 2 lines
This is because the qirqs[] array is under the machine and shared between both interrupt modes, xics and xive.
I don't see how that follows. ICSIRQState is indexed in terms of the ICS source number, not the global irq number, so I don't see why it has to match up with the qirq array.
The root cause is the use of
spapr->irq->nr_irqsto initialize the ICS and sPAPRXive object. In case of the 'dual' backend, it covers the full XIVE IRQ number space (0x2000today) but XICS only needs0x1000.
Quoted 7 lines
Quoted 3 lines
I think we can fix the offset issue by using the appropriate
nr_irqswhich should be for the XICS backend :spapr->irq->nr_irqs-ics->offsetSince the root cause is that the value of
spapr->irq->nr_irqsshould be different in XIVE and XICS, what about fixing it during reset ?
Nah this doesn't make sense :)
But if XICS always needs 0x1000, why just not change spapr_irq_init_xics() to use SPAPR_IRQ_XICS_NR_IRQS instead of spapr->irq->nr_irqs ?
Quoted 45 lines
Something like:
static void
spapr_irq_reset_dual(sPAPRMachineState *spapr, Error **errp){ [...] spapr->irq->nr_irqs = spapr_irq_current(spapr)->nr_irqs; spapr_irq_current(spapr)->reset(spapr, errp); }Quoted 32 lines
I keep in mind the XIVE support for nested guests and I think we will need to extend the IRQ number space in L1 and have the L2 use a portion of it (using an offset).
C.
Quoted 23 lines
Quoted 22 lines
Quoted 21 lines
Quoted 20 lines
hw/intc/xics_kvm.c +9
-0kvm_device_access (kernel_xics_fd,KVM_DEV_XICS_GRP_SOURCES , i + ics->offset, &state, false, &error_fatal);@@ -303,6 +312,15 @@ static int ics_set_kvm_state(ICSState *ics, int version_id)ICSIRQState *irq = &ics->irqs[i];int ret; +/* +* The KVM XICS device considers that the IRQ numbers should +* start at XICS_IRQ_BASE (0x1000). Ignore the lower 4K +* numbers (only claimed by XIVE for the CPU IPIs). +*/ +if (i + ics->offset <XICS_IRQ_BASE ) { +continue ; + } + state = irq->server; state |= (uint64_t )(irq->saved_priority &KVM_XICS_PRIORITY_MASK ) <<KVM_XICS_PRIORITY_SHIFT ;
PATCH 13/13 spapr: add KVM support to the 'dual' machine
In reply to Cédric Le GoaterThe interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)
hw/intc/spapr_xive.c +1 -7
@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );
hw/intc/spapr_xive_kvm.c +27 -0
@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len; +CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE +*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );
@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;
hw/intc/xics_kvm.c +25 -0
@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,
@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:
hw/intc/xive.c +0 -4
@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)
hw/ppc/spapr_irq.c +56 -23
142 lines not on this page · raw message
Signature · 1 line
2.20.1
Re: [Qemu-devel] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
Quoted 59 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 8 lines
hw/intc/spapr_xive_kvm.c +5
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len; +CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE
Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
Quoted 99 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-1@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ; -if (kvm_enabled ()) {
hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 10 lines
hw/ppc/spapr_irq.c +0
-10-if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;
I don't see anything that replaces the irqchip_required logic, which doesn't seem right.
Quoted 135 lines
hw/ppc/spapr_irq.c +56
-12+if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { + spapr->icp_type =TYPE_KVM_ICP ; + spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; } -error_free (local_err); - local_err =NULL ; }if (!spapr->ics) {@@ -119,10 +113,11 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) spapr->icp_type =TYPE_ICP ; spapr->ics =spapr_ics_create (spapr,TYPE_ICS_SIMPLE , nr_irqs, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } } - -error: -error_propagate (errp, local_err); }#define ICS_IRQ_FREE (ics, srcno) \@@ -233,7 +228,17 @@ static void spapr_irq_set_irq_xics(void *opaque, int srcno, int val)static void spapr_irq_reset_xics (sPAPRMachineState *spapr,Error **errp) { -/* TODO: create the KVM XICS device */ +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; + +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_init (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS connect failed: " ); +return ; + } + } }#define SPAPR_IRQ_XICS_NR_IRQS 0x1000@@ -393,6 +398,7 @@ static int spapr_irq_post_load_xive(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp) {CPUState *cs; +Error *local_err =NULL ;CPU_FOREACH (cs) {PowerPCCPU *cpu =POWERPC_CPU (cs);@@ -401,6 +407,15 @@ static void spapr_irq_reset_xive(sPAPRMachineState *spapr, Error **errp)spapr_xive_set_tctx_os_cam (cpu->tctx); } +if (kvmppc_xive_enabled ()) { +kvmppc_xive_connect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE connect failed: " ); +return ; + } + } +/* Activate the XIVE MMIOs */ spapr_xive_mmio_set_enabled (spapr->xive, true); }@@ -462,14 +477,8 @@ static sPAPRIrq *spapr_irq_current(sPAPRMachineState *spapr)static void spapr_irq_init_dual (sPAPRMachineState *spapr,Error **errp) { -MachineState *machine =MACHINE (spapr);Error *local_err =NULL ; -if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { -error_setg (errp,"No KVM support for the 'dual' machine" ); -return ; - } - spapr_irq_xics.init (spapr, &local_err);if (local_err) {error_propagate (errp, local_err);@@ -568,11 +577,16 @@ static void spapr_irq_cpu_intc_create_dual(sPAPRMachineState *spapr,static int spapr_irq_post_load_dual (sPAPRMachineState *spapr,int version_id) { +MachineState *machine =MACHINE (spapr); +/* * Force a reset of the XIVE backend after migration. The machine * defaults to XICS at startup. */ if (spapr_ovec_test (spapr->ov5_cas,OV5_XIVE_EXPLOIT )) { +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &error_fatal); + } spapr_irq_xive.reset (spapr, &error_fatal); }@@ -581,12 +595,31 @@ static int spapr_irq_post_load_dual(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_dual (sPAPRMachineState *spapr,Error **errp) { +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; +/* * Deactivate the XIVE MMIOs. The XIVE backend will reenable them * if selected. */ spapr_xive_mmio_set_enabled (spapr->xive, false); +/* Destroy all KVM devices */ +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS disconnect failed: " ); +return ; + } +kvmppc_xive_disconnect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE disconnect failed: " ); +return ; + } + } +spapr_irq_current (spapr)->reset (spapr, errp); }
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
Quoted 63 lines · David Gibson
Quoted 62 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len;
Quoted 8 lines
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +5
-0+CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
I guess you are right. I will check.
Quoted 98 lines
Quoted 98 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-0@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ;
Quoted 14 lines
hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 11 lines
hw/ppc/spapr_irq.c +0
-11-if (kvm_enabled ()) { -if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;I don't see anything that replaces the
irqchip_requiredlogic, which doesn't seem right.
Yes. We do loose the ability to fall back to the emulated device in case of failure. It is not impossible to do but it will require more changes to check what are the KVM capabilities before starting the machine.
Nevertheless, any failure in reset when setting the KVM backend will result in machine abort.
C.
Quoted 137 lines
Quoted 136 lines
hw/ppc/spapr_irq.c +56
-12+if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { + spapr->icp_type =TYPE_KVM_ICP ; + spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; } -error_free (local_err); - local_err =NULL ; }if (!spapr->ics) {@@ -119,10 +113,11 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) spapr->icp_type =TYPE_ICP ; spapr->ics =spapr_ics_create (spapr,TYPE_ICS_SIMPLE , nr_irqs, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } } - -error: -error_propagate (errp, local_err); }#define ICS_IRQ_FREE (ics, srcno) \@@ -233,7 +228,17 @@ static void spapr_irq_set_irq_xics(void *opaque, int srcno, int val)static void spapr_irq_reset_xics (sPAPRMachineState *spapr,Error **errp) { -/* TODO: create the KVM XICS device */ +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; + +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_init (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS connect failed: " ); +return ; + } + } }#define SPAPR_IRQ_XICS_NR_IRQS 0x1000@@ -393,6 +398,7 @@ static int spapr_irq_post_load_xive(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp) {CPUState *cs; +Error *local_err =NULL ;CPU_FOREACH (cs) {PowerPCCPU *cpu =POWERPC_CPU (cs);@@ -401,6 +407,15 @@ static void spapr_irq_reset_xive(sPAPRMachineState *spapr, Error **errp)spapr_xive_set_tctx_os_cam (cpu->tctx); } +if (kvmppc_xive_enabled ()) { +kvmppc_xive_connect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE connect failed: " ); +return ; + } + } +/* Activate the XIVE MMIOs */ spapr_xive_mmio_set_enabled (spapr->xive, true); }@@ -462,14 +477,8 @@ static sPAPRIrq *spapr_irq_current(sPAPRMachineState *spapr)static void spapr_irq_init_dual (sPAPRMachineState *spapr,Error **errp) { -MachineState *machine =MACHINE (spapr);Error *local_err =NULL ; -if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { -error_setg (errp,"No KVM support for the 'dual' machine" ); -return ; - } - spapr_irq_xics.init (spapr, &local_err);if (local_err) {error_propagate (errp, local_err);@@ -568,11 +577,16 @@ static void spapr_irq_cpu_intc_create_dual(sPAPRMachineState *spapr,static int spapr_irq_post_load_dual (sPAPRMachineState *spapr,int version_id) { +MachineState *machine =MACHINE (spapr); +/* * Force a reset of the XIVE backend after migration. The machine * defaults to XICS at startup. */ if (spapr_ovec_test (spapr->ov5_cas,OV5_XIVE_EXPLOIT )) { +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &error_fatal); + } spapr_irq_xive.reset (spapr, &error_fatal); }@@ -581,12 +595,31 @@ static int spapr_irq_post_load_dual(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_dual (sPAPRMachineState *spapr,Error **errp) { +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; +/* * Deactivate the XIVE MMIOs. The XIVE backend will reenable them * if selected. */ spapr_xive_mmio_set_enabled (spapr->xive, false); +/* Destroy all KVM devices */ +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS disconnect failed: " ); +return ; + } +kvmppc_xive_disconnect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE disconnect failed: " ); +return ; + } + } +spapr_irq_current (spapr)->reset (spapr, errp); }
Re: [Qemu-devel] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
Quoted 188 lines · Cédric Le Goater
Quoted 63 lines · David Gibson
Quoted 62 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len;Quoted 8 lines
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +5
-0+CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
I guess you are right. I will check.
Quoted 98 lines
Quoted 98 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-0@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ;Quoted 14 lines
hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 11 lines
hw/ppc/spapr_irq.c +0
-11-if (kvm_enabled ()) { -if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;I don't see anything that replaces the
irqchip_requiredlogic, which doesn't seem right.
Quoted 3 lines
Yes. We do loose the ability to fall back to the emulated device in case of failure. It is not impossible to do but it will require more changes to check what are the KVM capabilities before starting the machine.
Uh... it seems more like it's the other way around. We'll always fall back to emulated, even if we've explicitly said on the command line that we don't want that.
Quoted 143 lines
Nevertheless, any failure in reset when setting the KVM backend will result in machine abort.
C.
Quoted 137 lines
Quoted 136 lines
hw/ppc/spapr_irq.c +56
-12+if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { + spapr->icp_type =TYPE_KVM_ICP ; + spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; } -error_free (local_err); - local_err =NULL ; }if (!spapr->ics) {@@ -119,10 +113,11 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) spapr->icp_type =TYPE_ICP ; spapr->ics =spapr_ics_create (spapr,TYPE_ICS_SIMPLE , nr_irqs, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } } - -error: -error_propagate (errp, local_err); }#define ICS_IRQ_FREE (ics, srcno) \@@ -233,7 +228,17 @@ static void spapr_irq_set_irq_xics(void *opaque, int srcno, int val)static void spapr_irq_reset_xics (sPAPRMachineState *spapr,Error **errp) { -/* TODO: create the KVM XICS device */ +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; + +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_init (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS connect failed: " ); +return ; + } + } }#define SPAPR_IRQ_XICS_NR_IRQS 0x1000@@ -393,6 +398,7 @@ static int spapr_irq_post_load_xive(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp) {CPUState *cs; +Error *local_err =NULL ;CPU_FOREACH (cs) {PowerPCCPU *cpu =POWERPC_CPU (cs);@@ -401,6 +407,15 @@ static void spapr_irq_reset_xive(sPAPRMachineState *spapr, Error **errp)spapr_xive_set_tctx_os_cam (cpu->tctx); } +if (kvmppc_xive_enabled ()) { +kvmppc_xive_connect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE connect failed: " ); +return ; + } + } +/* Activate the XIVE MMIOs */ spapr_xive_mmio_set_enabled (spapr->xive, true); }@@ -462,14 +477,8 @@ static sPAPRIrq *spapr_irq_current(sPAPRMachineState *spapr)static void spapr_irq_init_dual (sPAPRMachineState *spapr,Error **errp) { -MachineState *machine =MACHINE (spapr);Error *local_err =NULL ; -if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { -error_setg (errp,"No KVM support for the 'dual' machine" ); -return ; - } - spapr_irq_xics.init (spapr, &local_err);if (local_err) {error_propagate (errp, local_err);@@ -568,11 +577,16 @@ static void spapr_irq_cpu_intc_create_dual(sPAPRMachineState *spapr,static int spapr_irq_post_load_dual (sPAPRMachineState *spapr,int version_id) { +MachineState *machine =MACHINE (spapr); +/* * Force a reset of the XIVE backend after migration. The machine * defaults to XICS at startup. */ if (spapr_ovec_test (spapr->ov5_cas,OV5_XIVE_EXPLOIT )) { +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &error_fatal); + } spapr_irq_xive.reset (spapr, &error_fatal); }@@ -581,12 +595,31 @@ static int spapr_irq_post_load_dual(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_dual (sPAPRMachineState *spapr,Error **errp) { +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; +/* * Deactivate the XIVE MMIOs. The XIVE backend will reenable them * if selected. */ spapr_xive_mmio_set_enabled (spapr->xive, false); +/* Destroy all KVM devices */ +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS disconnect failed: " ); +return ; + } +kvmppc_xive_disconnect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE disconnect failed: " ); +return ; + } + } +spapr_irq_current (spapr)->reset (spapr, errp); }
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
Quoted 189 lines · David Gibson
Quoted 188 lines · Cédric Le Goater
Quoted 63 lines · David Gibson
Quoted 62 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len;Quoted 8 lines
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +5
-0+CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
I guess you are right. I will check.
Quoted 112 lines
Quoted 99 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-1@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ; -if (kvm_enabled ()) {hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 10 lines
hw/ppc/spapr_irq.c +0
-10-if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;I don't see anything that replaces the
irqchip_requiredlogic, which doesn't seem right.
Quoted 7 lines
Quoted 3 lines
Yes. We do loose the ability to fall back to the emulated device in case of failure. It is not impossible to do but it will require more changes to check what are the KVM capabilities before starting the machine.
Uh... it seems more like it's the other way around. We'll always fall back to emulated, even if we've explicitly said on the command line that we don't want that.
Ah yes. The init function might be also broken.
XICS mode is a bit more difficult to handle than XIVE because we have different object type for the KVM device and the QEMU emulated device, and with the 'dual' mode, we activate the device at CAS reset time.
Failures being handled at reset time, should we keep the same logic and abort the machine at reset if the kernel irqchip is required ?
But we won't be able to fall back on the QEMU emulated device if KVM XICS fails and if the kernel irqchip is only allowed. It should work for XIVE though.
Thanks,
C.
Quoted 144 lines
Quoted 143 lines
Nevertheless, any failure in reset when setting the KVM backend will result in machine abort.
C.
Quoted 137 lines
Quoted 136 lines
hw/ppc/spapr_irq.c +56
-12+if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { + spapr->icp_type =TYPE_KVM_ICP ; + spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; } -error_free (local_err); - local_err =NULL ; }if (!spapr->ics) {@@ -119,10 +113,11 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) spapr->icp_type =TYPE_ICP ; spapr->ics =spapr_ics_create (spapr,TYPE_ICS_SIMPLE , nr_irqs, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } } - -error: -error_propagate (errp, local_err); }#define ICS_IRQ_FREE (ics, srcno) \@@ -233,7 +228,17 @@ static void spapr_irq_set_irq_xics(void *opaque, int srcno, int val)static void spapr_irq_reset_xics (sPAPRMachineState *spapr,Error **errp) { -/* TODO: create the KVM XICS device */ +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; + +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_init (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS connect failed: " ); +return ; + } + } }#define SPAPR_IRQ_XICS_NR_IRQS 0x1000@@ -393,6 +398,7 @@ static int spapr_irq_post_load_xive(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp) {CPUState *cs; +Error *local_err =NULL ;CPU_FOREACH (cs) {PowerPCCPU *cpu =POWERPC_CPU (cs);@@ -401,6 +407,15 @@ static void spapr_irq_reset_xive(sPAPRMachineState *spapr, Error **errp)spapr_xive_set_tctx_os_cam (cpu->tctx); } +if (kvmppc_xive_enabled ()) { +kvmppc_xive_connect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE connect failed: " ); +return ; + } + } +/* Activate the XIVE MMIOs */ spapr_xive_mmio_set_enabled (spapr->xive, true); }@@ -462,14 +477,8 @@ static sPAPRIrq *spapr_irq_current(sPAPRMachineState *spapr)static void spapr_irq_init_dual (sPAPRMachineState *spapr,Error **errp) { -MachineState *machine =MACHINE (spapr);Error *local_err =NULL ; -if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { -error_setg (errp,"No KVM support for the 'dual' machine" ); -return ; - } - spapr_irq_xics.init (spapr, &local_err);if (local_err) {error_propagate (errp, local_err);@@ -568,11 +577,16 @@ static void spapr_irq_cpu_intc_create_dual(sPAPRMachineState *spapr,static int spapr_irq_post_load_dual (sPAPRMachineState *spapr,int version_id) { +MachineState *machine =MACHINE (spapr); +/* * Force a reset of the XIVE backend after migration. The machine * defaults to XICS at startup. */ if (spapr_ovec_test (spapr->ov5_cas,OV5_XIVE_EXPLOIT )) { +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &error_fatal); + } spapr_irq_xive.reset (spapr, &error_fatal); }@@ -581,12 +595,31 @@ static int spapr_irq_post_load_dual(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_dual (sPAPRMachineState *spapr,Error **errp) { +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; +/* * Deactivate the XIVE MMIOs. The XIVE backend will reenable them * if selected. */ spapr_xive_mmio_set_enabled (spapr->xive, false); +/* Destroy all KVM devices */ +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS disconnect failed: " ); +return ; + } +kvmppc_xive_disconnect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE disconnect failed: " ); +return ; + } + } +spapr_irq_current (spapr)->reset (spapr, errp); }
Re: [Qemu-devel] [Qemu-ppc] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
Quoted 194 lines · On Wed, 13 Feb 2019 09:22:46 +0100 Cédric Le Goater <[email protected]> wrote:
Quoted 193 lines · David Gibson
Quoted 191 lines · Cédric Le Goater
Quoted 63 lines · David Gibson
Quoted 62 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len;Quoted 8 lines
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +5
-0+CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
I guess you are right. I will check.
Quoted 98 lines
Quoted 98 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-0@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ;Quoted 14 lines
hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 11 lines
hw/ppc/spapr_irq.c +0
-11-if (kvm_enabled ()) { -if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;I don't see anything that replaces the
irqchip_requiredlogic, which doesn't seem right.Yes. We do loose the ability to fall back to the emulated device in case of failure. It is not impossible to do but it will require more changes to check what are the KVM capabilities before starting the machine.
Quoted 8 lines
Quoted 3 lines
Uh... it seems more like it's the other way around. We'll always fall back to emulated, even if we've explicitly said on the command line that we don't want that.
Ah yes. The init function might be also broken.
XICS mode is a bit more difficult to handle than XIVE because we have different object type for the KVM device and the QEMU emulated device,
This is indeed a bit unfortunate, but I think there's still room for improvement. Let's look at the base classes:
struct ICPStateClass {
DeviceClass parent_class;
DeviceRealize parent_realize;
DeviceReset parent_reset;
void (*pre_save)(ICPState *icp);
int (*post_load)(ICPState *icp, int version_id);
void (*synchronize_state)(ICPState *icp);
};struct ICSStateClass {
DeviceClass parent_class;
DeviceRealize parent_realize;
DeviceReset parent_reset;
void (*pre_save)(ICSState *s);
int (*post_load)(ICSState *s, int version_id);
void (*reject)(ICSState *s, uint32_t irq);
void (*resend)(ICSState *s);
void (*eoi)(ICSState *s, uint32_t irq);
void (*synchronize_state)(ICSState *s);
};The pre_save and post_load callbacks are only used with the KVM device. They could be explicitely called from the corresponding VMStateDescription callbacks with a kvm_enabled() && kvm_irqchip_in_kernel() check.
Same goes for the synchronize_state callbacks, which are only needed for 'info pic'.
The reject, resend and eoi callbacks are only called by code that belongs to the QEMU emulated device. Either the RTAS/hypercalls or from the machine code with explicit checks like:
static void spapr_irq_set_irq_xics(void *opaque, int srcno, int val)
{
sPAPRMachineState *spapr = opaque;
MachineState *machine = MACHINE(opaque);
if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) {
ics_kvm_set_irq(spapr->ics, srcno, val);
} else {
ics_simple_set_irq(spapr->ics, srcno, val);
}
}or
static int spapr_irq_post_load_xics(sPAPRMachineState *spapr, int version_id)
{
if (!object_dynamic_cast(OBJECT(spapr->ics), TYPE_ICS_KVM)) {
CPUState *cs;
CPU_FOREACH(cs) {
PowerPCCPU *cpu = POWERPC_CPU(cs);
icp_resend(spapr_cpu_state(cpu)->icp);
}
}
return 0;
}Unless I'm missing something, the reject, resend and eoi callbacks could simply be removed. This would allow to unify KVM and QEMU emulation in the same ICP and ICS object types.
If this makes sense to you, I can have a look (already started actually ;-)
Quoted 5 lines
and with the 'dual' mode, we activate the device at CAS reset time.
Failures being handled at reset time, should we keep the same logic and abort the machine at reset if the kernel irqchip is required ?
If the user passed ic-mode=dual,kernel-irqchip=on, we should at least make sure KVM supports both XICS and XIVE devices during machine init. Then during reset if something goes wrong with KVM, it seems ok to abort.
If the user didn't pass kernel-irqchip, ie, kernel_irqchip_allowed is true and kernel_irqchip_required is false, the current behavior for XICS is to try KVM first and fallback to QEMU emulation. I guess it could be the same for XIVE.
Quoted 155 lines
But we won't be able to fall back on the QEMU emulated device if KVM XICS fails and if the kernel irqchip is only allowed. It should work for XIVE though.
Thanks,
C.
Quoted 144 lines
Quoted 143 lines
Nevertheless, any failure in reset when setting the KVM backend will result in machine abort.
C.
Quoted 137 lines
Quoted 136 lines
hw/ppc/spapr_irq.c +56
-12+if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { + spapr->icp_type =TYPE_KVM_ICP ; + spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, + &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; } -error_free (local_err); - local_err =NULL ; }if (!spapr->ics) {@@ -119,10 +113,11 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) spapr->icp_type =TYPE_ICP ; spapr->ics =spapr_ics_create (spapr,TYPE_ICS_SIMPLE , nr_irqs, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } } - -error: -error_propagate (errp, local_err); }#define ICS_IRQ_FREE (ics, srcno) \@@ -233,7 +228,17 @@ static void spapr_irq_set_irq_xics(void *opaque, int srcno, int val)static void spapr_irq_reset_xics (sPAPRMachineState *spapr,Error **errp) { -/* TODO: create the KVM XICS device */ +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; + +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_init (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS connect failed: " ); +return ; + } + } }#define SPAPR_IRQ_XICS_NR_IRQS 0x1000@@ -393,6 +398,7 @@ static int spapr_irq_post_load_xive(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_xive (sPAPRMachineState *spapr,Error **errp) {CPUState *cs; +Error *local_err =NULL ;CPU_FOREACH (cs) {PowerPCCPU *cpu =POWERPC_CPU (cs);@@ -401,6 +407,15 @@ static void spapr_irq_reset_xive(sPAPRMachineState *spapr, Error **errp)spapr_xive_set_tctx_os_cam (cpu->tctx); } +if (kvmppc_xive_enabled ()) { +kvmppc_xive_connect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE connect failed: " ); +return ; + } + } +/* Activate the XIVE MMIOs */ spapr_xive_mmio_set_enabled (spapr->xive, true); }@@ -462,14 +477,8 @@ static sPAPRIrq *spapr_irq_current(sPAPRMachineState *spapr)static void spapr_irq_init_dual (sPAPRMachineState *spapr,Error **errp) { -MachineState *machine =MACHINE (spapr);Error *local_err =NULL ; -if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { -error_setg (errp,"No KVM support for the 'dual' machine" ); -return ; - } - spapr_irq_xics.init (spapr, &local_err);if (local_err) {error_propagate (errp, local_err);@@ -568,11 +577,16 @@ static void spapr_irq_cpu_intc_create_dual(sPAPRMachineState *spapr,static int spapr_irq_post_load_dual (sPAPRMachineState *spapr,int version_id) { +MachineState *machine =MACHINE (spapr); +/* * Force a reset of the XIVE backend after migration. The machine * defaults to XICS at startup. */ if (spapr_ovec_test (spapr->ov5_cas,OV5_XIVE_EXPLOIT )) { +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &error_fatal); + } spapr_irq_xive.reset (spapr, &error_fatal); }@@ -581,12 +595,31 @@ static int spapr_irq_post_load_dual(sPAPRMachineState *spapr, int version_id)static void spapr_irq_reset_dual (sPAPRMachineState *spapr,Error **errp) { +MachineState *machine =MACHINE (spapr); +Error *local_err =NULL ; +/* * Deactivate the XIVE MMIOs. The XIVE backend will reenable them * if selected. */ spapr_xive_mmio_set_enabled (spapr->xive, false); +/* Destroy all KVM devices */ +if (kvm_enabled () &&machine_kernel_irqchip_allowed (machine)) { +xics_kvm_disconnect (spapr, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XICS disconnect failed: " ); +return ; + } +kvmppc_xive_disconnect (spapr->xive, &local_err); +if (local_err) { +error_propagate (errp, local_err); +error_prepend (errp,"KVM XIVE disconnect failed: " ); +return ; + } + } +spapr_irq_current (spapr)->reset (spapr, errp); }
Re: [Qemu-devel] [Qemu-ppc] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
Quoted 272 lines · Greg Kurz
Quoted 194 lines · On Wed, 13 Feb 2019 09:22:46 +0100 Cédric Le Goater <[email protected]> wrote:
Quoted 193 lines · David Gibson
Quoted 191 lines · Cédric Le Goater
Quoted 63 lines · David Gibson
Quoted 62 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len;Quoted 8 lines
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +5
-0+CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
I guess you are right. I will check.
Quoted 98 lines
Quoted 98 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-0@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ;Quoted 14 lines
hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 11 lines
hw/ppc/spapr_irq.c +0
-11-if (kvm_enabled ()) { -if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;I don't see anything that replaces the
irqchip_requiredlogic, which doesn't seem right.Yes. We do loose the ability to fall back to the emulated device in case of failure. It is not impossible to do but it will require more changes to check what are the KVM capabilities before starting the machine.
Quoted 8 lines
Quoted 3 lines
Uh... it seems more like it's the other way around. We'll always fall back to emulated, even if we've explicitly said on the command line that we don't want that.
Ah yes. The init function might be also broken.
XICS mode is a bit more difficult to handle than XIVE because we have different object type for the KVM device and the QEMU emulated device,
This is indeed a bit unfortunate, but I think there's still room for improvement. Let's look at the base classes:
struct ICPStateClass { DeviceClass parent_class; DeviceRealize parent_realize; DeviceReset parent_reset; void (*pre_save)(ICPState *icp); int (*post_load)(ICPState *icp, int version_id); void (*synchronize_state)(ICPState *icp); };struct ICSStateClass { DeviceClass parent_class; DeviceRealize parent_realize; DeviceReset parent_reset; void (*pre_save)(ICSState *s); int (*post_load)(ICSState *s, int version_id); void (*reject)(ICSState *s, uint32_t irq); void (*resend)(ICSState *s); void (*eoi)(ICSState *s, uint32_t irq); void (*synchronize_state)(ICSState *s); };The
pre_saveandpost_loadcallbacks are only used with the KVM device. They could be explicitely called from the corresponding VMStateDescription callbacks with akvm_enabled()&&kvm_irqchip_in_kernel()check.Same goes for the
synchronize_statecallbacks, which are only needed for 'info pic'.The reject, resend and eoi callbacks are only called by code that belongs to the QEMU emulated device. Either the RTAS/hypercalls or from the machine code with explicit checks like:
static void
spapr_irq_set_irq_xics(void *opaque, int srcno, int val){ sPAPRMachineState *spapr = opaque; MachineState *machine = MACHINE(opaque); if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) { ics_kvm_set_irq(spapr->ics, srcno, val); } else { ics_simple_set_irq(spapr->ics, srcno, val); } }or
static int
spapr_irq_post_load_xics(sPAPRMachineState *spapr, int version_id){ if (!object_dynamic_cast(OBJECT(spapr->ics), TYPE_ICS_KVM)) { CPUState *cs; CPU_FOREACH(cs) { PowerPCCPU *cpu = POWERPC_CPU(cs); icp_resend(spapr_cpu_state(cpu)->icp); } } return 0; }
Quoted 5 lines
Unless I'm missing something, the reject, resend and eoi callbacks could simply be removed. This would allow to unify KVM and QEMU emulation in the same ICP and ICS object types.
If this makes sense to you, I can have a look (already started actually ;-)
Please do. The use of different object types was something that seemed like a good idea at the time, but in hindsight, wasn't. In general different device types should represent guest-visibly different objects, not just implementation differences.
Quoted 6 lines
Quoted 5 lines
and with the 'dual' mode, we activate the device at CAS reset time.
Failures being handled at reset time, should we keep the same logic and abort the machine at reset if the kernel irqchip is required ?
Quoted 8 lines
If the user passed ic-mode=dual,kernel-irqchip=on, we should at least make sure KVM supports both XICS and XIVE devices during machine init. Then during reset if something goes wrong with KVM, it seems ok to abort.
If the user didn't pass kernel-irqchip, ie,
kernel_irqchip_allowedis true andkernel_irqchip_requiredis false, the current behavior for XICS is to try KVM first and fallback to QEMU emulation. I guess it could be the same for XIVE.
Yes, I think that's the behaviour we want, on all counts.
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [Qemu-ppc] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
Quoted 235 lines · David Gibson
Quoted 234 lines · Greg Kurz
Quoted 194 lines · On Wed, 13 Feb 2019 09:22:46 +0100 Cédric Le Goater <[email protected]> wrote:
Quoted 193 lines · David Gibson
Quoted 191 lines · Cédric Le Goater
Quoted 63 lines · David Gibson
Quoted 62 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len;Quoted 8 lines
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +5
-0+CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
I guess you are right. I will check.
Quoted 98 lines
Quoted 98 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-0@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ;Quoted 14 lines
hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 11 lines
hw/ppc/spapr_irq.c +0
-11-if (kvm_enabled ()) { -if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;I don't see anything that replaces the
irqchip_requiredlogic, which doesn't seem right.Yes. We do loose the ability to fall back to the emulated device in case of failure. It is not impossible to do but it will require more changes to check what are the KVM capabilities before starting the machine.
Quoted 8 lines
Quoted 3 lines
Uh... it seems more like it's the other way around. We'll always fall back to emulated, even if we've explicitly said on the command line that we don't want that.
Ah yes. The init function might be also broken.
XICS mode is a bit more difficult to handle than XIVE because we have different object type for the KVM device and the QEMU emulated device,
This is indeed a bit unfortunate, but I think there's still room for improvement. Let's look at the base classes:
struct ICPStateClass { DeviceClass parent_class; DeviceRealize parent_realize; DeviceReset parent_reset; void (*pre_save)(ICPState *icp); int (*post_load)(ICPState *icp, int version_id); void (*synchronize_state)(ICPState *icp); };struct ICSStateClass { DeviceClass parent_class; DeviceRealize parent_realize; DeviceReset parent_reset; void (*pre_save)(ICSState *s); int (*post_load)(ICSState *s, int version_id); void (*reject)(ICSState *s, uint32_t irq); void (*resend)(ICSState *s); void (*eoi)(ICSState *s, uint32_t irq); void (*synchronize_state)(ICSState *s); };
Quoted 4 lines
Quoted 4 lines
The
pre_saveandpost_loadcallbacks are only used with the KVM device. They could be explicitely called from the corresponding VMStateDescription callbacks with akvm_enabled()&&kvm_irqchip_in_kernel()check.
yes.
Quoted 2 lines
Quoted 2 lines
Same goes for the
synchronize_statecallbacks, which are only needed for 'info pic'.
yes, like we do for XIVE.
Quoted 30 lines
Quoted 30 lines
The reject, resend and eoi callbacks are only called by code that belongs to the QEMU emulated device. Either the RTAS/hypercalls or from the machine code with explicit checks like:
static void
spapr_irq_set_irq_xics(void *opaque, int srcno, int val){ sPAPRMachineState *spapr = opaque; MachineState *machine = MACHINE(opaque); if (kvm_enabled() && machine_kernel_irqchip_allowed(machine)) { ics_kvm_set_irq(spapr->ics, srcno, val); } else { ics_simple_set_irq(spapr->ics, srcno, val); } }or
static int
spapr_irq_post_load_xics(sPAPRMachineState *spapr, int version_id){ if (!object_dynamic_cast(OBJECT(spapr->ics), TYPE_ICS_KVM)) { CPUState *cs; CPU_FOREACH(cs) { PowerPCCPU *cpu = POWERPC_CPU(cs); icp_resend(spapr_cpu_state(cpu)->icp); } } return 0; }
Quoted 3 lines
Quoted 3 lines
Unless I'm missing something, the reject, resend and eoi callbacks could simply be removed. This would allow to unify KVM and QEMU emulation in the same ICP and ICS object types.
yes.
Quoted 6 lines
Quoted 1 line
If this makes sense to you, I can have a look (already started actually ;-)
Please do. The use of different object types was something that seemed like a good idea at the time, but in hindsight, wasn't. In general different device types should represent guest-visibly different objects, not just implementation differences.
yes. The direction we took to add KVM support in XIVE proved to be much simpler.
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines
and with the 'dual' mode, we activate the device at CAS reset time.
Failures being handled at reset time, should we keep the same logic and abort the machine at reset if the kernel irqchip is required ?
If the user passed ic-mode=dual,kernel-irqchip=on, we should at least make sure KVM supports both XICS and XIVE devices during machine init. Then during reset if something goes wrong with KVM, it seems ok to abort.
Quoted 6 lines
Quoted 4 lines
If the user didn't pass kernel-irqchip, ie,
kernel_irqchip_allowedis true andkernel_irqchip_requiredis false, the current behavior for XICS is to try KVM first and fallback to QEMU emulation. I guess it could be the same for XIVE.Yes, I think that's the behaviour we want, on all counts.
Thanks,
C.
Re: [Qemu-devel] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
In reply to Cédric Le GoaterQuoted 198 lines · Cédric Le Goater
Quoted 189 lines · David Gibson
Quoted 188 lines · Cédric Le Goater
Quoted 63 lines · David Gibson
Quoted 62 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len;Quoted 8 lines
hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 5 lines
hw/intc/spapr_xive_kvm.c +5
-0+CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
I guess you are right. I will check.
Quoted 112 lines
Quoted 99 lines
hw/intc/spapr_xive_kvm.c +22
-0+*/ +if (xive->fd !=-1 ) { +return ; + }if (!kvm_enabled () || !kvmppc_has_cap_xive ()) {error_setg (errp,"IRQ_XIVE capability must be present for KVM" );@@ -800,6 +809,24 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) xive->change =qemu_add_vm_change_state_handler ( kvmppc_xive_change_state_handler, xive); +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +kvmppc_xive_cpu_connect (cpu->tctx, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + +/* Update the KVM sources */ +kvmppc_xive_source_reset (xsrc, &local_err); +if (local_err) { +error_propagate (errp, local_err); +return ; + } + kvm_kernel_irqchip = true; kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true;hw/intc/xics_kvm.c +25
-0@@ -448,6 +448,16 @@ static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,int xics_kvm_init (sPAPRMachineState *spapr,Error **errp) {int rc; +CPUState *cs; +Error *local_err =NULL ; + +/* +* The KVM XICS device already in use. This is the case when +* rebooting XICS -> XICS +*/ +if (kernel_xics_fd !=-1 ) { +return 0 ; + }if (!kvm_enabled () || !kvm_check_extension (kvm_state,KVM_CAP_IRQ_XICS )) { error_setg(errp,@@ -496,6 +506,21 @@ int xics_kvm_init(sPAPRMachineState *spapr, Error **errp) kvm_msi_via_irqfd_allowed = true; kvm_gsi_direct_mapping = true; +/* Connect the presenters to the initial VCPUs of the machine */ +CPU_FOREACH (cs) { +PowerPCCPU *cpu =POWERPC_CPU (cs); + +icp_kvm_connect (cpu->icp, &local_err); +if (local_err) { +error_propagate (errp, local_err); + goto fail; + } +icp_set_kvm_state (cpu->icp,1 ); + } + +/* Update the KVM sources */ +ics_set_kvm_state (ICS_KVM (spapr->ics),1 ); +return 0 ; fail:hw/intc/xive.c +0
-4@@ -932,10 +932,6 @@ static void xive_source_reset(void *dev)/* PQs are initialized to 0b01 (Q=1) which corresponds to "ints off" */ memset (xsrc->status,XIVE_ESB_OFF , xsrc->nr_irqs); - -if (kvmppc_xive_enabled ()) { -kvmppc_xive_source_reset (xsrc, &error_fatal); - } }static void xive_source_realize (DeviceState *dev,Error **errp)hw/ppc/spapr_irq.c +0
-1@@ -98,20 +98,14 @@ static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp)int nr_irqs = spapr->irq->nr_irqs;Error *local_err =NULL ; -if (kvm_enabled ()) {hw/ppc/spapr_irq.c static void spapr_irq_init_xics(sPAPRMachineState *spapr, Error **errp) · line 111
Quoted 10 lines
hw/ppc/spapr_irq.c +0
-10-if (machine_kernel_irqchip_allowed (machine) && - !xics_kvm_init (spapr, &local_err)) { - spapr->icp_type =TYPE_KVM_ICP ; - spapr->ics =spapr_ics_create (spapr,TYPE_ICS_KVM , nr_irqs, - &local_err); - } -if (machine_kernel_irqchip_required (machine) && !spapr->ics) { -error_prepend (&local_err, -"kernel_irqchip requested but unavailable: " ); - goto error;I don't see anything that replaces the
irqchip_requiredlogic, which doesn't seem right.Quoted 7 lines
Quoted 3 lines
Yes. We do loose the ability to fall back to the emulated device in case of failure. It is not impossible to do but it will require more changes to check what are the KVM capabilities before starting the machine.
Uh... it seems more like it's the other way around. We'll always fall back to emulated, even if we've explicitly said on the command line that we don't want that.
Quoted 5 lines
Ah yes. The init function might be also broken.
XICS mode is a bit more difficult to handle than XIVE because we have different object type for the KVM device and the QEMU emulated device, and with the 'dual' mode, we activate the device at CAS reset time.
Yeah.. we should probably fix that.
Quoted 6 lines
Failures being handled at reset time, should we keep the same logic and abort the machine at reset if the kernel irqchip is required ?
But we won't be able to fall back on the QEMU emulated device if KVM XICS fails and if the kernel irqchip is only allowed. It should work for XIVE though.
That's fine. If we've said that kernel irqchip is required, we shouldn't fall back to emulation.
Signature · 4 lines
David Gibson | I'll have my music baroque, and my code david AT gibson.dropbear.id.au | minimalist, thank you. NOT _the_ _other_ | _way_ _around_! http://www.ozlabs.org/~dgibson
Re: [Qemu-devel] [PATCH 13/13] spapr: add KVM support to the 'dual' machine
In reply to David GibsonQuoted 74 lines · David Gibson
Quoted 73 lines · Cédric Le Goater
Quoted 71 lines · David Gibson
Quoted 59 lines · Cédric Le Goater
The interrupt mode is chosen by the CAS negotiation process and activated after a reset to take into account the required changes in the machine. This brings new constraints on how the associated KVM IRQ device is initialized.
Currently, each model takes care of the initialization of the KVM device in their realize method but this is not possible anymore as the initialization needs to be done globaly when the interrupt mode is known, i.e. when machine is reseted. It also means that we need a way to delete a KVM device when another mode is chosen.
Also, to support migration, the QEMU objects holding the state to transfer should always be available but not necessarily activated.
The overall approach of this proposal is to initialize both interrupt mode at the QEMU level and keep the IRQ number space in sync to allow switching from one mode to another. For the KVM side of things, the whole initialization of the KVM device, sources and presenters, is grouped in a single routine. The XICS and XIVE sPAPR IRQ reset handlers are modified accordingly to handle the init and the delete sequences of the KVM device.
As KVM is now initialized at reset, we loose the possiblity to fallback to the QEMU emulated mode in case of failure and failures become fatal to the machine.
Signed-off-by Cédric Le Goater <[email protected]>--- hw/intc/spapr_xive.c | 8 +--- hw/intc/spapr_xive_kvm.c | 27 ++++++++++++++ hw/intc/xics_kvm.c | 25 +++++++++++++ hw/intc/xive.c | 4 -- hw/ppc/spapr_irq.c | 79 ++++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 34 deletions(-)hw/intc/spapr_xive.c +1
-7@@ -330,13 +330,7 @@ static void spapr_xive_realize(DeviceState *dev, Error **errp) xive->eat =g_new0 (XiveEAS, xive->nr_irqs); xive->endt =g_new0 (XiveEND, xive->nr_ends); -if (kvmppc_xive_enabled ()) { -kvmppc_xive_connect (xive, &local_err); -if (local_err) { -error_propagate (errp, local_err); -return ; - } - }else { +if (!kvmppc_xive_enabled ()) {/* TIMA initialization */ memory_region_init_io (&xive->tm_mmio,OBJECT (xive), &xive_tm_ops, xive,"xive.tima" ,4ull <<TM_SHIFT );hw/intc/spapr_xive_kvm.c +0
-0hw/intc/spapr_xive_kvm.c void kvmppc_xive_connect(sPAPRXive *xive, Error **errp) · line 744
Quoted 8 lines
hw/intc/spapr_xive_kvm.c +5
-0@@ -737,6 +737,15 @@ void kvmppc_xive_connect(sPAPRXive *xive, Error **errp)Error *local_err =NULL ;size_t esb_len;size_t tima_len; +CPUState *cs; + +/* +* The KVM XIVE device already in use. This is the case when +* rebooting XIVE -> XIVE Can this case actually occur? Further down you appear to unconditionally destroy both KVM devices at reset time.
Quoted 1 line
Quoted 1 line
I guess you are right. I will check.
So we have to keep it for ic-mode=xive.
The number of test combinations is exploding. We now have :
3 hypervisors KVM, KVM nested, QEMU TCG 3 Interrupt modes xics, xive, dual (xics+xive) 3 kernel irqchip modes off, allow, required.
C.