aarch64: Add stolen time support
PATCH 1/1 aarch64: Add stolen time support
This patch add support for stolen time by sharing a memory region with the guest which will be used by the hypervisor to store the stolen time information. The exact format of the structure stored by the hypervisor is described in the ARM DEN0057A document.
--- Makefile | 3 +- arm/aarch64/arm-cpu.c | 2 + arm/aarch64/include/kvm/pvtime.h | 6 +++ arm/aarch64/pvtime.c | 83 +++++++++++++++++++++++++++++++ arm/include/arm-common/kvm-arch.h | 6 +++ arm/kvm-cpu.c | 14 +++--- 6 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 arm/aarch64/include/kvm/pvtime.h create mode 100644 arm/aarch64/pvtime.c
Makefile +2 -1
@@ -182,6 +182,7 @@ ifeq ($(ARCH), arm64) OBJS += arm/aarch64/arm-cpu.o OBJS += arm/aarch64/kvm-cpu.o OBJS += arm/aarch64/kvm.o + OBJS += arm/aarch64/pvtime.o ARCH_INCLUDE := $(HDRS_ARM_COMMON) ARCH_INCLUDE += -Iarm/aarch64/include
@@ -582,4 +583,4 @@ ifneq ($(MAKECMDGOALS),clean) KVMTOOLS-VERSION-FILE: @$(SHELL_PATH) util/KVMTOOLS-VERSION-GEN $(OUTPUT) -endif +endif
arm/aarch64/arm-cpu.c +2 -0
@@ -2,6 +2,7 @@#include "kvm/kvm.h" #include "kvm/kvm-cpu.h" #include "kvm/util.h" +#include "kvm/pvtime.h" #include "arm-common/gic.h" #include "arm-common/timer.h"
@@ -22,6 +23,7 @@ static void generate_fdt_nodes(void *fdt, struct kvm *kvm)static int arm_cpu__vcpu_init (struct kvm_cpu *vcpu) { vcpu->generate_fdt_nodes = generate_fdt_nodes; +pvtime__setup_vcpu (vcpu);return 0 ; }
arm/aarch64/include/kvm/pvtime.h new file +6 -0
@@ -0,0 +1,6 @@ +#ifndef KVM__PVTIME_H +#define KVM__PVTIME_H + +void pvtime__setup_vcpu (struct kvm_cpu *vcpu); + +#endif /* KVM__PVTIME_H */
arm/aarch64/pvtime.c new file +83 -0
83 lines not on this page · raw message
arm/include/arm-common/kvm-arch.h +6 -0
@@ -54,6 +54,12 @@#define ARM_PCI_MMIO_SIZE (ARM_MEMORY_AREA - \ (ARM_AXI_AREA + ARM_PCI_CFG_SIZE)) +#define AARCH64_PVTIME_IPA_MAX_SIZE (0x10000) +#define AARCH64_PROTECTED_VM_FW_MAX_SIZE (0x200000) +#define AARCH64_PVTIME_IPA_START (ARM_MEMORY_AREA - \ + AARCH64_PROTECTED_VM_FW_MAX_SIZE - \ + AARCH64_PVTIME_IPA_MAX_SIZE) +#define AARCH64_PVTIME_SIZE (64)#define ARM_LOMAP_MAX_MEMORY ((1ULL << 32) - ARM_MEMORY_AREA)#define ARM_HIMAP_MAX_MEMORY ((1ULL << 40) - ARM_MEMORY_AREA)
arm/kvm-cpu.c +7 -7
@@ -116,6 +116,13 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)die ("Unable to find matching target" ); } +/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );
@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE ); -/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );
Signature · 1 line
2.35.1.265.g69c8d7142f-goog
Marc Zyngier
maintainerHi Sebastian,
Thanks for looking into this. A few comments below.
Quoted 30 lines · Sebastian Ene
This patch add support for stolen time by sharing a memory region with the guest which will be used by the hypervisor to store the stolen time information. The exact format of the structure stored by the hypervisor is described in the ARM DEN0057A document.
Signed-off-by Sebastian Ene <[email protected]>--- Makefile | 3 +- arm/aarch64/arm-cpu.c | 2 + arm/aarch64/include/kvm/pvtime.h | 6 +++ arm/aarch64/pvtime.c | 83 +++++++++++++++++++++++++++++++ arm/include/arm-common/kvm-arch.h | 6 +++ arm/kvm-cpu.c | 14 +++--- 6 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 arm/aarch64/include/kvm/pvtime.h create mode 100644 arm/aarch64/pvtime.cMakefile +1
-0@@ -182,6 +182,7 @@ ifeq ($(ARCH), arm64) OBJS += arm/aarch64/arm-cpu.o OBJS += arm/aarch64/kvm-cpu.o OBJS += arm/aarch64/kvm.o + OBJS += arm/aarch64/pvtime.o ARCH_INCLUDE := $(HDRS_ARM_COMMON) ARCH_INCLUDE += -Iarm/aarch64/include
Makefile ifneq ($(MAKECMDGOALS),clean) · line 586
Quoted 6 lines
Makefile +1
-1@@ -582,4 +583,4 @@ ifneq ($(MAKECMDGOALS),clean) KVMTOOLS-VERSION-FILE: @$(SHELL_PATH) util/KVMTOOLS-VERSION-GEN $(OUTPUT) -endif +endif
Spurious change?
Quoted 27 lines
arm/aarch64/arm-cpu.c +2
-0@@ -2,6 +2,7 @@#include "kvm/kvm.h" #include "kvm/kvm-cpu.h" #include "kvm/util.h" +#include "kvm/pvtime.h" #include "arm-common/gic.h" #include "arm-common/timer.h" @@ -22,6 +23,7 @@ static void generate_fdt_nodes(void *fdt, struct kvm *kvm)static int arm_cpu__vcpu_init (struct kvm_cpu *vcpu) { vcpu->generate_fdt_nodes = generate_fdt_nodes; +pvtime__setup_vcpu (vcpu);return 0 ; }arm/aarch64/include/kvm/pvtime.h new file +0
-0
arm/aarch64/include/kvm/pvtime.h line 6
Quoted 6 lines
arm/aarch64/include/kvm/pvtime.h new file +6
-0@@ -0,0 +1,6 @@ +#ifndef KVM__PVTIME_H +#define KVM__PVTIME_H + +void pvtime__setup_vcpu (struct kvm_cpu *vcpu); + +#endif /* KVM__PVTIME_H */
How about sticking this in kvm-cpu-arch.h instead? A whole new include file for just a prototype isn't totally warranted.
Quoted 15 lines
arm/aarch64/pvtime.c new file +9
-0@@ -0,0 +1,83 @@ +#include "kvm/kvm.h" +#include "kvm/kvm-cpu.h" +#include "kvm/util.h" +#include "kvm/pvtime.h" + +#include <linux/byteorder.h> +#include <linux/types.h> + +struct pvtime_data_priv {
arm/aarch64/pvtime.c line 19
Quoted 10 lines
arm/aarch64/pvtime.c new file +10
-0+bool is_supported; +char *usr_mem; +}; + +static struct pvtime_data_priv pvtime_data = { + .is_supported = true, + .usr_mem =NULL +}; + +static int pvtime__aloc_region (struct kvm *kvm)
s/aloc/alloc/ ?
Quoted 41 lines
arm/aarch64/pvtime.c new file +41
-0+{ +char *mem; +int ret =0 ; + + mem =mmap (NULL ,AARCH64_PVTIME_IPA_MAX_SIZE ,PROT_RW , +MAP_ANON_NORESERVE ,-1 ,0 ); +if (mem ==MAP_FAILED ) +return -ENOMEM ; + + ret =kvm__register_dev_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , mem); +if (ret) { +munmap (mem,AARCH64_PVTIME_IPA_MAX_SIZE ); +return ret; + } + + pvtime_data.usr_mem = mem; +return ret; +} + +static int pvtime__teardown_region (struct kvm *kvm) +{ +kvm__destroy_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , pvtime_data.usr_mem); +munmap (pvtime_data.usr_mem,AARCH64_PVTIME_IPA_MAX_SIZE ); + pvtime_data.usr_mem =NULL ; +return 0 ; +} + +void pvtime__setup_vcpu (struct kvm_cpu *vcpu) +{ +int ret; +u64 pvtime_guest_addr =AARCH64_PVTIME_IPA_START + vcpu->cpu_id * +AARCH64_PVTIME_SIZE ; +struct kvm_device_attr pvtime_attr = (struct kvm_device_attr ) { + .group =KVM_ARM_VCPU_PVTIME_CTRL , + .addr =KVM_ARM_VCPU_PVTIME_IPA + }; + +if (!pvtime_data.is_supported) +return ;
arm/aarch64/pvtime.c line 70
Quoted 10 lines
arm/aarch64/pvtime.c new file +10
-0+ +if (!pvtime_data.usr_mem) { + ret =pvtime__aloc_region (vcpu->kvm); +if (ret) + goto out_err_alloc; + } + + ret =ioctl (vcpu->vcpu_fd,KVM_HAS_DEVICE_ATTR , &pvtime_attr); +if (ret) + goto out_err_attr;
You should check for the stolen time capability before allocating and mapping the memory.
Quoted 19 lines
arm/aarch64/pvtime.c new file +13
-0+ + pvtime_attr.addr = (u64)&pvtime_guest_addr; + ret =ioctl (vcpu->vcpu_fd,KVM_SET_DEVICE_ATTR , &pvtime_attr); +if (!ret) +return ; + +out_err_attr: +pvtime__teardown_region (vcpu->kvm); +out_err_alloc: + pvtime_data.is_supported = false; +} + +dev_exit (pvtime__teardown_region);arm/include/arm-common/kvm-arch.h +0
-0
arm/include/arm-common/kvm-arch.h line 57
Quoted 4 lines
arm/include/arm-common/kvm-arch.h +1
-0@@ -54,6 +54,12 @@#define ARM_PCI_MMIO_SIZE (ARM_MEMORY_AREA - \ (ARM_AXI_AREA + ARM_PCI_CFG_SIZE)) +#define AARCH64_PVTIME_IPA_MAX_SIZE (0x10000)
SZ_64K?
arm/include/arm-common/kvm-arch.h line 58
Quoted 1 line
arm/include/arm-common/kvm-arch.h +1
-0+#define AARCH64_PROTECTED_VM_FW_MAX_SIZE (0x200000)
This definitely looks like something that shouldn't be there. Yet.
Quoted 28 lines
arm/include/arm-common/kvm-arch.h +4
-0+#define AARCH64_PVTIME_IPA_START (ARM_MEMORY_AREA - \ + AARCH64_PROTECTED_VM_FW_MAX_SIZE - \ + AARCH64_PVTIME_IPA_MAX_SIZE) +#define AARCH64_PVTIME_SIZE (64)#define ARM_LOMAP_MAX_MEMORY ((1ULL << 32) - ARM_MEMORY_AREA)#define ARM_HIMAP_MAX_MEMORY ((1ULL << 40) - ARM_MEMORY_AREA)arm/kvm-cpu.c +7
-0@@ -116,6 +116,13 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) die("Unable to find matching target"); } +/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm · line 135
Quoted 10 lines
arm/kvm-cpu.c +0
-7vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE ); -/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -
What is the reason for moving these assignments around?
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) · line 136
Quoted 2 lines
arm/kvm-cpu.c +0
-0if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );
Thanks,
M.
Signature · 5 lines
Jazz is not dead. It just smells funny... _______________________________________________ kvmarm mailing list [email protected] https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Quoted 4 lines · Marc Zyngier
Hi Sebastian,
Thanks for looking into this. A few comments below.
Hello Marc,
Thanks for taking the time to review my patch.
Quoted 32 lines · Sebastian Ene
Quoted 31 lines
This patch add support for stolen time by sharing a memory region with the guest which will be used by the hypervisor to store the stolen time information. The exact format of the structure stored by the hypervisor is described in the ARM DEN0057A document.
Signed-off-by Sebastian Ene <[email protected]>--- Makefile | 3 +- arm/aarch64/arm-cpu.c | 2 + arm/aarch64/include/kvm/pvtime.h | 6 +++ arm/aarch64/pvtime.c | 83 +++++++++++++++++++++++++++++++ arm/include/arm-common/kvm-arch.h | 6 +++ arm/kvm-cpu.c | 14 +++--- 6 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 arm/aarch64/include/kvm/pvtime.h create mode 100644 arm/aarch64/pvtime.cMakefile +1
-0@@ -182,6 +182,7 @@ ifeq ($(ARCH), arm64) OBJS += arm/aarch64/arm-cpu.o OBJS += arm/aarch64/kvm-cpu.o OBJS += arm/aarch64/kvm.o + OBJS += arm/aarch64/pvtime.o ARCH_INCLUDE := $(HDRS_ARM_COMMON) ARCH_INCLUDE += -Iarm/aarch64/include@@ -582,4 +583,4 @@ ifneq ($(MAKECMDGOALS),clean)
Quoted 8 lines
Makefile ifneq ($(MAKECMDGOALS),clean) · line 586
Quoted 5 lines
Makefile +1
-1KVMTOOLS-VERSION-FILE: @$(SHELL_PATH) util/KVMTOOLS-VERSION-GEN $(OUTPUT) -endif +endifSpurious change?
Right, ACK I will revert this change.
Quoted 21 lines
Quoted 21 lines
arm/aarch64/arm-cpu.c +2
-0@@ -2,6 +2,7 @@#include "kvm/kvm.h" #include "kvm/kvm-cpu.h" #include "kvm/util.h" +#include "kvm/pvtime.h" #include "arm-common/gic.h" #include "arm-common/timer.h" @@ -22,6 +23,7 @@ static void generate_fdt_nodes(void *fdt, struct kvm *kvm)static int arm_cpu__vcpu_init (struct kvm_cpu *vcpu) { vcpu->generate_fdt_nodes = generate_fdt_nodes; +pvtime__setup_vcpu (vcpu);return 0 ; }
Quoted 17 lines
arm/aarch64/include/kvm/pvtime.h line 6
Quoted 13 lines
arm/aarch64/include/kvm/pvtime.h new file +6
-0@@ -0,0 +1,6 @@ +#ifndef KVM__PVTIME_H +#define KVM__PVTIME_H + +void pvtime__setup_vcpu (struct kvm_cpu *vcpu); + +#endif /* KVM__PVTIME_H */ How about sticking this in
kvm-cpu-arch.hinstead? A whole new include file for just a prototype isn't totally warranted.
I think that's a good ideea, I will do this.
Quoted 19 lines
Quoted 19 lines
arm/aarch64/pvtime.c new file +13
-0@@ -0,0 +1,83 @@ +#include "kvm/kvm.h" +#include "kvm/kvm-cpu.h" +#include "kvm/util.h" +#include "kvm/pvtime.h" + +#include <linux/byteorder.h> +#include <linux/types.h> + +struct pvtime_data_priv { +bool is_supported; +char *usr_mem; +}; +
Quoted 9 lines
arm/aarch64/pvtime.c line 19
Quoted 6 lines
arm/aarch64/pvtime.c new file +6
-0+static struct pvtime_data_priv pvtime_data = { + .is_supported = true, + .usr_mem =NULL +}; + +static int pvtime__aloc_region (struct kvm *kvm)s/aloc/alloc/ ?
I will fix the typo.
Quoted 47 lines
Quoted 47 lines
arm/aarch64/pvtime.c new file +47
-0+{ +char *mem; +int ret =0 ; + + mem =mmap (NULL ,AARCH64_PVTIME_IPA_MAX_SIZE ,PROT_RW , +MAP_ANON_NORESERVE ,-1 ,0 ); +if (mem ==MAP_FAILED ) +return -ENOMEM ; + + ret =kvm__register_dev_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , mem); +if (ret) { +munmap (mem,AARCH64_PVTIME_IPA_MAX_SIZE ); +return ret; + } + + pvtime_data.usr_mem = mem; +return ret; +} + +static int pvtime__teardown_region (struct kvm *kvm) +{ +kvm__destroy_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , pvtime_data.usr_mem); +munmap (pvtime_data.usr_mem,AARCH64_PVTIME_IPA_MAX_SIZE ); + pvtime_data.usr_mem =NULL ; +return 0 ; +} + +void pvtime__setup_vcpu (struct kvm_cpu *vcpu) +{ +int ret; +u64 pvtime_guest_addr =AARCH64_PVTIME_IPA_START + vcpu->cpu_id * +AARCH64_PVTIME_SIZE ; +struct kvm_device_attr pvtime_attr = (struct kvm_device_attr ) { + .group =KVM_ARM_VCPU_PVTIME_CTRL , + .addr =KVM_ARM_VCPU_PVTIME_IPA + }; + +if (!pvtime_data.is_supported) +return ; + +if (!pvtime_data.usr_mem) { + ret =pvtime__aloc_region (vcpu->kvm); +if (ret) + goto out_err_alloc; + }
Quoted 9 lines
arm/aarch64/pvtime.c line 70
Quoted 4 lines
arm/aarch64/pvtime.c new file +4
-0+ + ret =ioctl (vcpu->vcpu_fd,KVM_HAS_DEVICE_ATTR , &pvtime_attr); +if (ret) + goto out_err_attr;You should check for the stolen time capability before allocating and mapping
the memory.
Good catch ! I will do this, thanks.
Quoted 22 lines
Quoted 22 lines
arm/aarch64/pvtime.c new file +13
-0+ + pvtime_attr.addr = (u64)&pvtime_guest_addr; + ret =ioctl (vcpu->vcpu_fd,KVM_SET_DEVICE_ATTR , &pvtime_attr); +if (!ret) +return ; + +out_err_attr: +pvtime__teardown_region (vcpu->kvm); +out_err_alloc: + pvtime_data.is_supported = false; +} + +dev_exit (pvtime__teardown_region);arm/include/arm-common/kvm-arch.h +0
-0@@ -54,6 +54,12 @@#define ARM_PCI_MMIO_SIZE (ARM_MEMORY_AREA - \ (ARM_AXI_AREA + ARM_PCI_CFG_SIZE))
Quoted 4 lines
arm/include/arm-common/kvm-arch.h line 57
Quoted 1 line
arm/include/arm-common/kvm-arch.h +1
-0+#define AARCH64_PVTIME_IPA_MAX_SIZE (0x10000)
SZ_64K?
I will update the AARCH64_PVTIME_IPA_MAX_SIZE defintion with SZ_64K.
Quoted 4 lines
arm/include/arm-common/kvm-arch.h line 58
Quoted 1 line
arm/include/arm-common/kvm-arch.h +1
-0+#define AARCH64_PROTECTED_VM_FW_MAX_SIZE (0x200000)This definitely looks like something that shouldn't be there. Yet.
I agree, I will remove it for the moment.
Quoted 31 lines
Quoted 31 lines
arm/include/arm-common/kvm-arch.h +4
-0+#define AARCH64_PVTIME_IPA_START (ARM_MEMORY_AREA - \ + AARCH64_PROTECTED_VM_FW_MAX_SIZE - \ + AARCH64_PVTIME_IPA_MAX_SIZE) +#define AARCH64_PVTIME_SIZE (64)#define ARM_LOMAP_MAX_MEMORY ((1ULL << 32) - ARM_MEMORY_AREA)#define ARM_HIMAP_MAX_MEMORY ((1ULL << 40) - ARM_MEMORY_AREA)arm/kvm-cpu.c +7
-0@@ -116,6 +116,13 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) die("Unable to find matching target"); } +/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE );
Quoted 10 lines
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) · line 134
Quoted 7 lines
arm/kvm-cpu.c +0
-7-/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -What is the reason for moving these assignments around?
The setup of the pvtime is done during the vcpu initialisation and I need to access the kvm structure during that time.
Quoted 8 lines
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) · line 136
Quoted 2 lines
arm/kvm-cpu.c +0
-0if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );Thanks,
M.Signature · 1 line
Jazz is not dead. It just smells funny...