Implement CPU hotplug on Arm
v2 cover Implement CPU hotplug on Arm
This series implements support for CPU hotplug/unplug on Arm. To achieve this, several things need to be done:
1. XEN_SYSCTL_CPU_HOTPLUG_* calls implemented.
2. timer and GIC maintenance interrupts switched to static irqactions to remove the need for freeing them during release_irq.
3. Enabled the build of xen-hptool on Arm.
Tested on QEMU.
v1->v2:
* see individual patches
Mykyta Poturai (4):
arm/time: Use static irqaction
arm/gic: Use static irqaction
arm/sysctl: Implement cpu hotplug ops
tools: Allow building xen-hptool without CONFIG_MIGRATE
config/Tools.mk.in | 1 + tools/configure | 30 +++++++++++++++++++++ tools/configure.ac | 1 + tools/libs/guest/Makefile.common | 4 +++ tools/misc/Makefile | 2 +- xen/arch/arm/gic.c | 11 ++++++-- xen/arch/arm/sysctl.c | 45 ++++++++++++++++++++++++++++++++ xen/arch/arm/time.c | 21 ++++++++++++--- 8 files changed, 108 insertions(+), 7 deletions(-) mode change 100755 => 100644 tools/configure
Signature · 1 line
2.34.1
PATCH v2 1/4 arm/time: Use static irqaction
When stopping a core deinit_timer_interrupt is called in non-alloc context, which causes xfree in release_irq to fail an assert.
To fix this, switch to a statically allocated irqaction that does not need to be freed in release_irq.
v1->v2:
* Use percpu actions
--- xen/arch/arm/time.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-)
xen/arch/arm/time.c +17 -4
@@ -303,9 +303,15 @@ static void check_timer_irq_cfg(unsigned int irq, const char *which) "WARNING: %s-timer IRQ%u is not level triggered.\n", which, irq); } +DEFINE_PER_CPU_READ_MOSTLY(struct irqaction, irq_hyp); +DEFINE_PER_CPU_READ_MOSTLY(struct irqaction, irq_virt); +/* Set up the timer interrupt on this CPU */ void init_timer_interrupt (void ) { +struct irqaction *hyp_action = &this_cpu (irq_hyp); +struct irqaction *virt_action = &this_cpu (irq_virt); +/* Sensible defaults */ WRITE_SYSREG64 (0 ,CNTVOFF_EL2 );/* No VM-specific offset */ /* Do not let the VMs program the physical timer, only read the physical counter */
@@ -314,10 +320,17 @@ void init_timer_interrupt(void)WRITE_SYSREG (0 ,CNTHP_CTL_EL2 );/* Hypervisor's timer disabled */ isb (); -request_irq (timer_irq[TIMER_HYP_PPI ],0 , htimer_interrupt, -"hyptimer" ,NULL ); -request_irq (timer_irq[TIMER_VIRT_PPI ],0 , vtimer_interrupt, -"virtimer" ,NULL ); + hyp_action->name ="hyptimer" ; + hyp_action->handler = htimer_interrupt; + hyp_action->dev_id =NULL ; + hyp_action->free_on_release =0 ; +setup_irq (timer_irq[TIMER_HYP_PPI ],0 , hyp_action); + + virt_action->name ="virtimer" ; + virt_action->handler = vtimer_interrupt; + virt_action->dev_id =NULL ; + virt_action->free_on_release =0 ; +setup_irq (timer_irq[TIMER_VIRT_PPI ],0 , virt_action);check_timer_irq_cfg (timer_irq[TIMER_HYP_PPI ],"hypervisor" );check_timer_irq_cfg (timer_irq[TIMER_VIRT_PPI ],"virtual" );
Signature · 1 line
2.34.1
PATCH v2 2/4 arm/gic: Use static irqaction
In reply to Mykyta PoturaiWhen stopping a core cpu_gic_callback is called in non-alloc context, which causes xfree in release_irq to fail an assert.
To fix this, switch to a statically allocated irqaction that does not need to be freed in release_irq.
v1->v2:
* use percpu actions
--- xen/arch/arm/gic.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)
xen/arch/arm/gic.c +9 -2
@@ -386,10 +386,17 @@ void gic_dump_info(struct vcpu *v) gic_hw_ops->dump_state (v); } +DEFINE_PER_CPU_READ_MOSTLY(struct irqaction, irq_maintenance); +void init_maintenance_interrupt (void ) { -request_irq (gic_hw_ops->info->maintenance_irq,0 , maintenance_interrupt, -"irq-maintenance" ,NULL ); +struct irqaction *maintenance = &this_cpu (irq_maintenance); + + maintenance->name ="irq-maintenance" ; + maintenance->handler = maintenance_interrupt; + maintenance->dev_id =NULL ; + maintenance->free_on_release =0 ; +setup_irq (gic_hw_ops->info->maintenance_irq,0 , maintenance); } int gic_make_hwdom_dt_node(const struct domain *d,
Signature · 1 line
2.34.1
PATCH v2 3/4 arm/sysctl: Implement cpu hotplug ops
In reply to Mykyta PoturaiImplement XEN_SYSCTL_CPU_HOTPLUG_{ONLINE,OFFLINE} calls to allow for enabling/disabling CPU cores in runtime.
v1->v2:
* remove SMT ops
* remove cpu == 0 checks
* add XSM hooks
* only implement for 64bit Arm
--- xen/arch/arm/sysctl.c | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+)
xen/arch/arm/sysctl.c +45 -0
@@ -12,6 +12,8 @@#include <xen/dt-overlay.h> #include <xen/errno.h> #include <xen/hypercall.h> +#include <xen/cpu.h> +#include <xsm/xsm.h> #include <asm/arm64/sve.h> #include <public/sysctl.h>
@@ -23,6 +25,42 @@ void arch_do_physinfo(struct xen_sysctl_physinfo *pi) XEN_SYSCTL_PHYSCAP_ARM_SVE_MASK); } +#ifdef CONFIG_ARM_64 +static long cpu_up_helper (void *data) +{ +unsigned long cpu = (unsigned long ) data; +return cpu_up (cpu); +} + +static long cpu_down_helper (void *data) +{ +unsigned long cpu = (unsigned long ) data; +return cpu_down (cpu); +} + +static long cpu_hotplug_sysctl (struct xen_sysctl_cpu_hotplug *hotplug) +{ +int ret; + +switch (hotplug->op) { +case XEN_SYSCTL_CPU_HOTPLUG_ONLINE : + ret =xsm_resource_plug_core (XSM_HOOK ); +if ( ret ) +return ret; +return continue_hypercall_on_cpu (0 , cpu_up_helper,_p (hotplug->cpu)); + +case XEN_SYSCTL_CPU_HOTPLUG_OFFLINE : + ret =xsm_resource_unplug_core (XSM_HOOK ); +if ( ret ) +return ret; +return continue_hypercall_on_cpu (0 , cpu_down_helper,_p (hotplug->cpu)); + +default : +return -EOPNOTSUPP ; + } +} +#endif +long arch_do_sysctl (struct xen_sysctl *sysctl,XEN_GUEST_HANDLE_PARAM (xen_sysctl_t) u_sysctl) {
@@ -34,6 +72,13 @@ long arch_do_sysctl(struct xen_sysctl *sysctl, ret =dt_overlay_sysctl (&sysctl->u.dt_overlay);break ; +/* CPU Hotplug only implemented for 64-bit Arm */ +#ifdef CONFIG_ARM_64 +case XEN_SYSCTL_cpu_hotplug: + ret =cpu_hotplug_sysctl (&sysctl->u.cpu_hotplug); +break ; +#endif +default : ret = -ENOSYS ;break ;
Signature · 1 line
2.34.1
PATCH v2 4/4 tools: Allow building xen-hptool without CONFIG_MIGRATE
In reply to Mykyta PoturaiWith CPU hotplug sysctls implemented on Arm it becomes useful to have a tool for calling them. Introduce a new congifure option "hptool" to allow building hptool separately from other migration tools, and enable it by default.
v1->v2:
* switch to configure from legacy config
--- config/Tools.mk.in | 1 + tools/configure | 30 ++++++++++++++++++++++++++++++ tools/configure.ac | 1 + tools/libs/guest/Makefile.common | 4 ++++ tools/misc/Makefile | 2 +- 5 files changed, 37 insertions(+), 1 deletion(-) mode change 100755 => 100644 tools/configure
config/Tools.mk.in +1 -0
@@ -49,6 +49,7 @@ CONFIG_LIBNL := @libnl@ CONFIG_GOLANG := @golang@ CONFIG_PYGRUB := @pygrub@ CONFIG_LIBFSIMAGE := @libfsimage@ +CONFIG_HPTOOL := @hptool@ CONFIG_SYSTEMD := @systemd@ XEN_SYSTEMD_DIR := @SYSTEMD_DIR@
tools/configure +30 -0
@@ -728,6 +728,7 @@ LD86 AS86 ipxe LINUX_BACKEND_MODULES +hptool pygrub golang seabios
@@ -834,6 +835,7 @@ enable_ovmf enable_seabios enable_golang enable_pygrub +enable_hptool with_linux_backend_modules enable_ipxe with_system_ipxe
@@ -1519,6 +1521,7 @@ Optional Features: --disable-seabios Disable SeaBIOS (default is ENABLED) --disable-golang Disable Go tools (default is ENABLED) --disable-pygrub Disable pygrub (default is ENABLED) + --disable-hptool Disable hptool (default is ENABLED) --enable-ipxe Enable in-tree IPXE, (DEFAULT is off, see also --with-system-ipxe) --enable-rombios Enable ROMBIOS, (DEFAULT is on if ipxe is enabled,
@@ -4807,6 +4810,33 @@ pygrub=$ax_cv_pygrub +# Check whether --enable-hptool was given. +if test ${enable_hptool+y} +then : + enableval=$enable_hptool; +fi + + +if test "x$enable_hptool" = "xno" +then : + + ax_cv_hptool="n" + +elif test "x$enable_hptool" = "xyes" +then : + + ax_cv_hptool="y" + +elif test -z $ax_cv_hptool +then : + + ax_cv_hptool="y" + +fi +hptool=$ax_cv_hptool + + + # Check whether --with-linux-backend-modules was given. if test ${with_linux_backend_modules+y}
tools/configure.ac +1 -0
@@ -90,6 +90,7 @@ AX_ARG_DEFAULT_DISABLE([ovmf], [Enable OVMF]) AX_ARG_DEFAULT_ENABLE([seabios], [Disable SeaBIOS]) AX_ARG_DEFAULT_ENABLE([golang], [Disable Go tools]) AX_ARG_DEFAULT_ENABLE([pygrub], [Disable pygrub]) +AX_ARG_DEFAULT_ENABLE([hptool], [Disable hptool]) AC_ARG_WITH([linux-backend-modules], AS_HELP_STRING([--with-linux-backend-modules="mod1 mod2"],
tools/libs/guest/Makefile.common +4 -0
@@ -25,6 +25,10 @@ OBJS-y += xg_core.o OBJS-$(CONFIG_X86) += xg_core_x86.o OBJS-$(CONFIG_ARM) += xg_core_arm.o +ifneq (,$(filter y,$(CONFIG_MIGRATE)$(CONFIG_HPTOOL))) +OBJS-y += xg_offline_page.o +endif + vpath %.c ../../../xen/common/libelf LIBELF_OBJS += libelf-tools.o libelf-loader.o
tools/misc/Makefile +1 -1
@@ -16,7 +16,7 @@ INSTALL_BIN += xencov_split INSTALL_BIN += $(INSTALL_BIN-y) # Everything to be installed in regular sbin/ -INSTALL_SBIN-$(CONFIG_MIGRATE) += xen-hptool +INSTALL_SBIN-$(CONFIG_HPTOOL) += xen-hptool INSTALL_SBIN-$(CONFIG_X86) += xen-hvmcrash INSTALL_SBIN-$(CONFIG_X86) += xen-hvmctx INSTALL_SBIN-$(CONFIG_X86) += xen-lowmemd
Signature · 1 line
2.34.1