aarch64: Add stolen time support
PATCH v2 1/1 aarch64: Add stolen time support
This patch adds support for stolen time by sharing a memory region with the guest which will be used by the hypervisor to store the stolen time information. The exact format of the structure stored by the hypervisor is described in the ARM DEN0057A document.
--- Makefile | 1 + arm/aarch64/arm-cpu.c | 1 + arm/aarch64/include/kvm/kvm-cpu-arch.h | 1 + arm/aarch64/pvtime.c | 84 ++++++++++++++++++++++++++ arm/include/arm-common/kvm-arch.h | 4 ++ arm/kvm-cpu.c | 14 ++--- 6 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 arm/aarch64/pvtime.c
Makefile +1 -0
@@ -182,6 +182,7 @@ ifeq ($(ARCH), arm64) OBJS += arm/aarch64/arm-cpu.o OBJS += arm/aarch64/kvm-cpu.o OBJS += arm/aarch64/kvm.o + OBJS += arm/aarch64/pvtime.o ARCH_INCLUDE := $(HDRS_ARM_COMMON) ARCH_INCLUDE += -Iarm/aarch64/include
arm/aarch64/arm-cpu.c +1 -0
@@ -22,6 +22,7 @@ static void generate_fdt_nodes(void *fdt, struct kvm *kvm)static int arm_cpu__vcpu_init (struct kvm_cpu *vcpu) { vcpu->generate_fdt_nodes = generate_fdt_nodes; +kvm_cpu__setup_pvtime (vcpu);return 0 ; }
arm/aarch64/include/kvm/kvm-cpu-arch.h +1 -0
@@ -19,5 +19,6 @@void kvm_cpu__select_features (struct kvm *kvm,struct kvm_vcpu_init *init);int kvm_cpu__configure_features (struct kvm_cpu *vcpu); +void kvm_cpu__setup_pvtime (struct kvm_cpu *vcpu);#endif /* KVM__KVM_CPU_ARCH_H */
arm/aarch64/pvtime.c new file +84 -0
84 lines not on this page · raw message
arm/include/arm-common/kvm-arch.h +4 -0
@@ -54,6 +54,10 @@#define ARM_PCI_MMIO_SIZE (ARM_MEMORY_AREA - \ (ARM_AXI_AREA + ARM_PCI_CFG_SIZE)) +#define AARCH64_PVTIME_IPA_MAX_SIZE SZ_64K +#define AARCH64_PVTIME_IPA_START (ARM_MEMORY_AREA - \ + AARCH64_PVTIME_IPA_MAX_SIZE) +#define AARCH64_PVTIME_SIZE (64)#define ARM_LOMAP_MAX_MEMORY ((1ULL << 32) - ARM_MEMORY_AREA)#define ARM_HIMAP_MAX_MEMORY ((1ULL << 40) - ARM_MEMORY_AREA)
arm/kvm-cpu.c +7 -7
@@ -116,6 +116,13 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)die ("Unable to find matching target" ); } +/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );
@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE ); -/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );
Signature · 1 line
2.35.1.265.g69c8d7142f-goog
Hi,
Some general comments while I familiarize myself with the stolen time spec.
Quoted 7 lines · Sebastian Ene
This patch adds support for stolen time by sharing a memory region with the guest which will be used by the hypervisor to store the stolen time information. The exact format of the structure stored by the hypervisor is described in the ARM DEN0057A document.
Signed-off-by Sebastian Ene <[email protected]>---
It is customary to describe the changes you have made from the previous version, to make it easier for the people who want to review your code.
Quoted 139 lines
Makefile | 1 + arm/aarch64/arm-cpu.c | 1 + arm/aarch64/include/kvm/kvm-cpu-arch.h | 1 + arm/aarch64/pvtime.c | 84 ++++++++++++++++++++++++++ arm/include/arm-common/kvm-arch.h | 4 ++ arm/kvm-cpu.c | 14 ++--- 6 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 arm/aarch64/pvtime.cMakefile +1
-0@@ -182,6 +182,7 @@ ifeq ($(ARCH), arm64) OBJS += arm/aarch64/arm-cpu.o OBJS += arm/aarch64/kvm-cpu.o OBJS += arm/aarch64/kvm.o + OBJS += arm/aarch64/pvtime.o ARCH_INCLUDE := $(HDRS_ARM_COMMON) ARCH_INCLUDE += -Iarm/aarch64/includearm/aarch64/arm-cpu.c +1
-0@@ -22,6 +22,7 @@ static void generate_fdt_nodes(void *fdt, struct kvm *kvm)static int arm_cpu__vcpu_init (struct kvm_cpu *vcpu) { vcpu->generate_fdt_nodes = generate_fdt_nodes; +kvm_cpu__setup_pvtime (vcpu);return 0 ; }arm/aarch64/include/kvm/kvm-cpu-arch.h +1
-0@@ -19,5 +19,6 @@void kvm_cpu__select_features (struct kvm *kvm,struct kvm_vcpu_init *init);int kvm_cpu__configure_features (struct kvm_cpu *vcpu); +void kvm_cpu__setup_pvtime (struct kvm_cpu *vcpu);#endif /* KVM__KVM_CPU_ARCH_H */ arm/aarch64/pvtime.c new file +84
-0@@ -0,0 +1,84 @@ +#include "kvm/kvm.h" +#include "kvm/kvm-cpu.h" +#include "kvm/util.h" + +#include <linux/byteorder.h> +#include <linux/types.h> + +struct pvtime_data_priv { +bool is_supported; +char *usr_mem; +}; + +static struct pvtime_data_priv pvtime_data = { + .is_supported = true, + .usr_mem =NULL +}; + +static int pvtime__alloc_region (struct kvm *kvm) +{ +char *mem; +int ret =0 ; + + mem =mmap (NULL ,AARCH64_PVTIME_IPA_MAX_SIZE ,PROT_RW , +MAP_ANON_NORESERVE ,-1 ,0 ); +if (mem ==MAP_FAILED ) +return -ENOMEM ; + + ret =kvm__register_dev_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , mem); +if (ret) { +munmap (mem,AARCH64_PVTIME_IPA_MAX_SIZE ); +return ret; + } + + pvtime_data.usr_mem = mem; +return ret; +} + +static int pvtime__teardown_region (struct kvm *kvm) +{ +if (pvtime_data.usr_mem ==NULL ) +return 0 ; + +kvm__destroy_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , pvtime_data.usr_mem); +munmap (pvtime_data.usr_mem,AARCH64_PVTIME_IPA_MAX_SIZE ); + pvtime_data.usr_mem =NULL ; +return 0 ; +} + +void kvm_cpu__setup_pvtime (struct kvm_cpu *vcpu) +{ +int ret; +u64 pvtime_guest_addr =AARCH64_PVTIME_IPA_START + vcpu->cpu_id * +AARCH64_PVTIME_SIZE ; +struct kvm_device_attr pvtime_attr = (struct kvm_device_attr ) { + .group =KVM_ARM_VCPU_PVTIME_CTRL , + .addr =KVM_ARM_VCPU_PVTIME_IPA + }; + +if (!pvtime_data.is_supported) +return ; + + ret =ioctl (vcpu->vcpu_fd,KVM_HAS_DEVICE_ATTR , &pvtime_attr); +if (ret) + goto out_err; + +if (!pvtime_data.usr_mem) { + ret =pvtime__alloc_region (vcpu->kvm); +if (ret) + goto out_err; + } + + pvtime_attr.addr = (u64)&pvtime_guest_addr; + ret =ioctl (vcpu->vcpu_fd,KVM_SET_DEVICE_ATTR , &pvtime_attr); +if (!ret) +return ; + +pvtime__teardown_region (vcpu->kvm); +out_err: + pvtime_data.is_supported = false; +} + +dev_exit (pvtime__teardown_region);arm/include/arm-common/kvm-arch.h +0
-0
arm/include/arm-common/kvm-arch.h line 59
Quoted 6 lines
arm/include/arm-common/kvm-arch.h +3
-0@@ -54,6 +54,10 @@#define ARM_PCI_MMIO_SIZE (ARM_MEMORY_AREA - \ (ARM_AXI_AREA + ARM_PCI_CFG_SIZE)) +#define AARCH64_PVTIME_IPA_MAX_SIZE SZ_64K +#define AARCH64_PVTIME_IPA_START (ARM_MEMORY_AREA - \ + AARCH64_PVTIME_IPA_MAX_SIZE)
This overlaps with the ARM_PCI_MMIO_AREA. If you want to change the memory layout for a guest, there's a handy ASCII map at the top of this file. That should also be updated to reflect the modified layout.
Why do you want to put it below RAM? Is there a requirement to have it there or was the location chosen for other reasons?
arm/include/arm-common/kvm-arch.h line 60
Quoted 1 line
arm/include/arm-common/kvm-arch.h +1
-0+#define AARCH64_PVTIME_SIZE (64)
This looks like something that should go in pvtime.c, as it's not relevant to the memory layout.
Quoted 22 lines
arm/include/arm-common/kvm-arch.h +0
-0#define ARM_LOMAP_MAX_MEMORY ((1ULL << 32) - ARM_MEMORY_AREA)#define ARM_HIMAP_MAX_MEMORY ((1ULL << 40) - ARM_MEMORY_AREA)arm/kvm-cpu.c +7
-0@@ -116,6 +116,13 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)die ("Unable to find matching target" ); } +/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) · line 134
Quoted 10 lines
arm/kvm-cpu.c +0
-7@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE ); -/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -
Why this change?
Thanks,
Alex
Quoted 6 lines
arm/kvm-cpu.c +0
-0if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );Signature · 1 line
2.35.1.265.g69c8d7142f-goog
Hi,
Quoted 5 lines · Alexandru Elisei
Hi,
Some general comments while I familiarize myself with the stolen time spec.
On Thu, Feb 17, 2022 at 03:08:53PM +0000, Sebastian Ene wrote:
[..]
Quoted 15 lines
Quoted 15 lines
arm/kvm-cpu.c +7
-0+/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE );
Quoted 9 lines
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) · line 134
Quoted 7 lines
arm/kvm-cpu.c +0
-7-/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -Why this change?
Got it know, it's needed for target->init, which is actually arm_cpu__vcpu_init. Weird that it wasn't that way in the first place, but that's because of the awkwardness of generating the fdt nodes for the gic/timer/pmu.
Thanks,
Alex
Quoted 14 lines
Thanks,
AlexQuoted 6 lines
arm/kvm-cpu.c +0
-0if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );Signature · 1 line
2.35.1.265.g69c8d7142f-goog_______________________________________________
kvmarm mailing list
[email protected]
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Quoted 10 lines · Alexandru Elisei
Hi,
Some general comments while I familiarize myself with the stolen time spec.
Quoted 5 lines · Sebastian Ene
This patch adds support for stolen time by sharing a memory region with the guest which will be used by the hypervisor to store the stolen time information. The exact format of the structure stored by the hypervisor is described in the ARM DEN0057A document.
Quoted 6 lines
Quoted 2 lines
Signed-off-by Sebastian Ene <[email protected]>---
It is customary to describe the changes you have made from the previous version, to make it easier for the people who want to review your code.
Hello,
Thanks for the feedback. I will include the changelog in the next patches.
Quoted 142 lines
Quoted 142 lines
Makefile | 1 + arm/aarch64/arm-cpu.c | 1 + arm/aarch64/include/kvm/kvm-cpu-arch.h | 1 + arm/aarch64/pvtime.c | 84 ++++++++++++++++++++++++++ arm/include/arm-common/kvm-arch.h | 4 ++ arm/kvm-cpu.c | 14 ++--- 6 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 arm/aarch64/pvtime.cMakefile +1
-0@@ -182,6 +182,7 @@ ifeq ($(ARCH), arm64) OBJS += arm/aarch64/arm-cpu.o OBJS += arm/aarch64/kvm-cpu.o OBJS += arm/aarch64/kvm.o + OBJS += arm/aarch64/pvtime.o ARCH_INCLUDE := $(HDRS_ARM_COMMON) ARCH_INCLUDE += -Iarm/aarch64/includearm/aarch64/arm-cpu.c +1
-0@@ -22,6 +22,7 @@ static void generate_fdt_nodes(void *fdt, struct kvm *kvm)static int arm_cpu__vcpu_init (struct kvm_cpu *vcpu) { vcpu->generate_fdt_nodes = generate_fdt_nodes; +kvm_cpu__setup_pvtime (vcpu);return 0 ; }arm/aarch64/include/kvm/kvm-cpu-arch.h +1
-0@@ -19,5 +19,6 @@void kvm_cpu__select_features (struct kvm *kvm,struct kvm_vcpu_init *init);int kvm_cpu__configure_features (struct kvm_cpu *vcpu); +void kvm_cpu__setup_pvtime (struct kvm_cpu *vcpu);#endif /* KVM__KVM_CPU_ARCH_H */ arm/aarch64/pvtime.c new file +84
-0@@ -0,0 +1,84 @@ +#include "kvm/kvm.h" +#include "kvm/kvm-cpu.h" +#include "kvm/util.h" + +#include <linux/byteorder.h> +#include <linux/types.h> + +struct pvtime_data_priv { +bool is_supported; +char *usr_mem; +}; + +static struct pvtime_data_priv pvtime_data = { + .is_supported = true, + .usr_mem =NULL +}; + +static int pvtime__alloc_region (struct kvm *kvm) +{ +char *mem; +int ret =0 ; + + mem =mmap (NULL ,AARCH64_PVTIME_IPA_MAX_SIZE ,PROT_RW , +MAP_ANON_NORESERVE ,-1 ,0 ); +if (mem ==MAP_FAILED ) +return -ENOMEM ; + + ret =kvm__register_dev_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , mem); +if (ret) { +munmap (mem,AARCH64_PVTIME_IPA_MAX_SIZE ); +return ret; + } + + pvtime_data.usr_mem = mem; +return ret; +} + +static int pvtime__teardown_region (struct kvm *kvm) +{ +if (pvtime_data.usr_mem ==NULL ) +return 0 ; + +kvm__destroy_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , pvtime_data.usr_mem); +munmap (pvtime_data.usr_mem,AARCH64_PVTIME_IPA_MAX_SIZE ); + pvtime_data.usr_mem =NULL ; +return 0 ; +} + +void kvm_cpu__setup_pvtime (struct kvm_cpu *vcpu) +{ +int ret; +u64 pvtime_guest_addr =AARCH64_PVTIME_IPA_START + vcpu->cpu_id * +AARCH64_PVTIME_SIZE ; +struct kvm_device_attr pvtime_attr = (struct kvm_device_attr ) { + .group =KVM_ARM_VCPU_PVTIME_CTRL , + .addr =KVM_ARM_VCPU_PVTIME_IPA + }; + +if (!pvtime_data.is_supported) +return ; + + ret =ioctl (vcpu->vcpu_fd,KVM_HAS_DEVICE_ATTR , &pvtime_attr); +if (ret) + goto out_err; + +if (!pvtime_data.usr_mem) { + ret =pvtime__alloc_region (vcpu->kvm); +if (ret) + goto out_err; + } + + pvtime_attr.addr = (u64)&pvtime_guest_addr; + ret =ioctl (vcpu->vcpu_fd,KVM_SET_DEVICE_ATTR , &pvtime_attr); +if (!ret) +return ; + +pvtime__teardown_region (vcpu->kvm); +out_err: + pvtime_data.is_supported = false; +} + +dev_exit (pvtime__teardown_region);arm/include/arm-common/kvm-arch.h +0
-0@@ -54,6 +54,10 @@#define ARM_PCI_MMIO_SIZE (ARM_MEMORY_AREA - \ (ARM_AXI_AREA + ARM_PCI_CFG_SIZE))
Quoted 7 lines
arm/include/arm-common/kvm-arch.h line 59
Quoted 3 lines
arm/include/arm-common/kvm-arch.h +3
-0+#define AARCH64_PVTIME_IPA_MAX_SIZE SZ_64K +#define AARCH64_PVTIME_IPA_START (ARM_MEMORY_AREA - \ + AARCH64_PVTIME_IPA_MAX_SIZE)This overlaps with the
ARM_PCI_MMIO_AREA. If you want to change the memory layout for a guest, there's a handy ASCII map at the top of this file. That should also be updated to reflect the modified layout.
Right, when pvtime is supported it ovelaps that region. While this feature is supported only for arm64, does it make sense to update the ASCII map in this header which is arm-common ? Also, probably it makes sense to move all of the definitions to pvtime.c ? What do you think ?
Quoted 4 lines
Why do you want to put it below RAM? Is there a requirement to have it there or was the location chosen for other reasons?
I don't have a strong argument for this placement but I am open to suggestions if you would like to move it somewhere else. For example, we are placing pvtime in the same region in crosvm.
Quoted 5 lines
arm/include/arm-common/kvm-arch.h line 60
Quoted 1 line
arm/include/arm-common/kvm-arch.h +1
-0+#define AARCH64_PVTIME_SIZE (64)This looks like something that should go in
pvtime.c, as it's not relevant to the memory layout.
I agree, I will move this. Thanks.
Quoted 25 lines
Quoted 25 lines
arm/include/arm-common/kvm-arch.h +0
-0#define ARM_LOMAP_MAX_MEMORY ((1ULL << 32) - ARM_MEMORY_AREA)#define ARM_HIMAP_MAX_MEMORY ((1ULL << 40) - ARM_MEMORY_AREA)arm/kvm-cpu.c +7
-0@@ -116,6 +116,13 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)die ("Unable to find matching target" ); } +/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE );
Quoted 10 lines
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) · line 134
Quoted 7 lines
arm/kvm-cpu.c +0
-7-/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -Why this change?
I will add a comment to describe why I moved this. During pvtime setup, in vcpu initialization I need to access the kvm structure from the vcpu.
Quoted 3 lines
Thanks,
Alex
Thanks for the review,
Sebastian
Quoted 6 lines
Quoted 6 lines
arm/kvm-cpu.c +0
-0if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );Signature · 1 line
2.35.1.265.g69c8d7142f-goog
Hi,
Quoted 173 lines · Sebastian Ene
Quoted 10 lines · Alexandru Elisei
Hi,
Some general comments while I familiarize myself with the stolen time spec.
Quoted 5 lines · Sebastian Ene
This patch adds support for stolen time by sharing a memory region with the guest which will be used by the hypervisor to store the stolen time information. The exact format of the structure stored by the hypervisor is described in the ARM DEN0057A document.
Quoted 6 lines
Quoted 2 lines
Signed-off-by Sebastian Ene <[email protected]>---
It is customary to describe the changes you have made from the previous version, to make it easier for the people who want to review your code.
Hello,
Thanks for the feedback. I will include the changelog in the next patches.
Quoted 142 lines
Quoted 142 lines
Makefile | 1 + arm/aarch64/arm-cpu.c | 1 + arm/aarch64/include/kvm/kvm-cpu-arch.h | 1 + arm/aarch64/pvtime.c | 84 ++++++++++++++++++++++++++ arm/include/arm-common/kvm-arch.h | 4 ++ arm/kvm-cpu.c | 14 ++--- 6 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 arm/aarch64/pvtime.cMakefile +1
-0@@ -182,6 +182,7 @@ ifeq ($(ARCH), arm64) OBJS += arm/aarch64/arm-cpu.o OBJS += arm/aarch64/kvm-cpu.o OBJS += arm/aarch64/kvm.o + OBJS += arm/aarch64/pvtime.o ARCH_INCLUDE := $(HDRS_ARM_COMMON) ARCH_INCLUDE += -Iarm/aarch64/includearm/aarch64/arm-cpu.c +1
-0@@ -22,6 +22,7 @@ static void generate_fdt_nodes(void *fdt, struct kvm *kvm)static int arm_cpu__vcpu_init (struct kvm_cpu *vcpu) { vcpu->generate_fdt_nodes = generate_fdt_nodes; +kvm_cpu__setup_pvtime (vcpu);return 0 ; }arm/aarch64/include/kvm/kvm-cpu-arch.h +1
-0@@ -19,5 +19,6 @@void kvm_cpu__select_features (struct kvm *kvm,struct kvm_vcpu_init *init);int kvm_cpu__configure_features (struct kvm_cpu *vcpu); +void kvm_cpu__setup_pvtime (struct kvm_cpu *vcpu);#endif /* KVM__KVM_CPU_ARCH_H */ arm/aarch64/pvtime.c new file +84
-0@@ -0,0 +1,84 @@ +#include "kvm/kvm.h" +#include "kvm/kvm-cpu.h" +#include "kvm/util.h" + +#include <linux/byteorder.h> +#include <linux/types.h> + +struct pvtime_data_priv { +bool is_supported; +char *usr_mem; +}; + +static struct pvtime_data_priv pvtime_data = { + .is_supported = true, + .usr_mem =NULL +}; + +static int pvtime__alloc_region (struct kvm *kvm) +{ +char *mem; +int ret =0 ; + + mem =mmap (NULL ,AARCH64_PVTIME_IPA_MAX_SIZE ,PROT_RW , +MAP_ANON_NORESERVE ,-1 ,0 ); +if (mem ==MAP_FAILED ) +return -ENOMEM ; + + ret =kvm__register_dev_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , mem); +if (ret) { +munmap (mem,AARCH64_PVTIME_IPA_MAX_SIZE ); +return ret; + } + + pvtime_data.usr_mem = mem; +return ret; +} + +static int pvtime__teardown_region (struct kvm *kvm) +{ +if (pvtime_data.usr_mem ==NULL ) +return 0 ; + +kvm__destroy_mem (kvm,AARCH64_PVTIME_IPA_START , +AARCH64_PVTIME_IPA_MAX_SIZE , pvtime_data.usr_mem); +munmap (pvtime_data.usr_mem,AARCH64_PVTIME_IPA_MAX_SIZE ); + pvtime_data.usr_mem =NULL ; +return 0 ; +} + +void kvm_cpu__setup_pvtime (struct kvm_cpu *vcpu) +{ +int ret; +u64 pvtime_guest_addr =AARCH64_PVTIME_IPA_START + vcpu->cpu_id * +AARCH64_PVTIME_SIZE ; +struct kvm_device_attr pvtime_attr = (struct kvm_device_attr ) { + .group =KVM_ARM_VCPU_PVTIME_CTRL , + .addr =KVM_ARM_VCPU_PVTIME_IPA + }; + +if (!pvtime_data.is_supported) +return ; + + ret =ioctl (vcpu->vcpu_fd,KVM_HAS_DEVICE_ATTR , &pvtime_attr); +if (ret) + goto out_err; + +if (!pvtime_data.usr_mem) { + ret =pvtime__alloc_region (vcpu->kvm); +if (ret) + goto out_err; + } + + pvtime_attr.addr = (u64)&pvtime_guest_addr; + ret =ioctl (vcpu->vcpu_fd,KVM_SET_DEVICE_ATTR , &pvtime_attr); +if (!ret) +return ; + +pvtime__teardown_region (vcpu->kvm); +out_err: + pvtime_data.is_supported = false; +} + +dev_exit (pvtime__teardown_region);arm/include/arm-common/kvm-arch.h +0
-0@@ -54,6 +54,10 @@#define ARM_PCI_MMIO_SIZE (ARM_MEMORY_AREA - \ (ARM_AXI_AREA + ARM_PCI_CFG_SIZE))Quoted 7 lines
arm/include/arm-common/kvm-arch.h line 59
Quoted 3 lines
arm/include/arm-common/kvm-arch.h +3
-0+#define AARCH64_PVTIME_IPA_MAX_SIZE SZ_64K +#define AARCH64_PVTIME_IPA_START (ARM_MEMORY_AREA - \ + AARCH64_PVTIME_IPA_MAX_SIZE)This overlaps with the
ARM_PCI_MMIO_AREA. If you want to change the memory layout for a guest, there's a handy ASCII map at the top of this file. That should also be updated to reflect the modified layout.
Quoted 4 lines
Right, when pvtime is supported it ovelaps that region. While this feature is supported only for arm64, does it make sense to update the ASCII map in this header which is arm-common ? Also, probably it makes sense to move all of the definitions to
pvtime.c? What do you think ?
kvmtool uses the same memory layout for arm and arm64. If you want to have different memory layouts between arm and arm64, then the memory layout from this file should be moved to arm/aarch{32,64}/include/kvm/kvm-arch.h and modified there.
I'm not 100% sure how to proceed here. pvtime requires only 64KiB of memory out of a 1GiB - 256MiB region, so making ARM_PCI_MMIO_SIZE 64KiB smaller for both architectures is not a big deal as far as I can tell.
Having both arm and arm64 use the same memory layout means that fixes benefit both architectures (there had been several memory layout fixes in the past, where regions were overlapping).
On the other hand, KVM for arm does not exist anymore, and hasn't for quite some time, and changing the memory layout for an arm64 feature means that we risk breaking arm support, which is becoming harder and harder to test. And this will become increasingly likely as new arm64 features that change the memory layout are added.
To be honest, I think the best person to answer this is Will, as the maintainer of kvmtool. I have been going back and forth about this, I think I slightly prefer making the change here, to the common layout.
Quoted 9 lines
Quoted 5 lines
Why do you want to put it below RAM? Is there a requirement to have it there or was the location chosen for other reasons?
I don't have a strong argument for this placement but I am open to suggestions if you would like to move it somewhere else. For example, we are placing pvtime in the same region in crosvm.
It looks ok to me to have it there, clumped with the other reserved regions.
Quoted 45 lines
Quoted 6 lines
arm/include/arm-common/kvm-arch.h line 60
Quoted 2 lines
arm/include/arm-common/kvm-arch.h +1
-0+#define AARCH64_PVTIME_SIZE (64)This looks like something that should go in
pvtime.c, as it's not relevant to the memory layout.I agree, I will move this. Thanks.
Quoted 25 lines
Quoted 25 lines
arm/include/arm-common/kvm-arch.h +0
-0#define ARM_LOMAP_MAX_MEMORY ((1ULL << 32) - ARM_MEMORY_AREA)#define ARM_HIMAP_MAX_MEMORY ((1ULL << 40) - ARM_MEMORY_AREA)arm/kvm-cpu.c +7
-0@@ -116,6 +116,13 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)die ("Unable to find matching target" ); } +/* Populate the vcpu structure. */ + vcpu->kvm = kvm; + vcpu->cpu_id = cpu_id; + vcpu->cpu_type = vcpu_init.target; + vcpu->cpu_compatible = target->compatible; + vcpu->is_running = true; +if (err || target->init (vcpu))die ("Unable to initialise vcpu" );@@ -125,13 +132,6 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) vcpu->ring = (void *)vcpu->kvm_run + (coalesced_offset *PAGE_SIZE );Quoted 10 lines
arm/kvm-cpu.c struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id) · line 134
Quoted 7 lines
arm/kvm-cpu.c +0
-7-/* Populate the vcpu structure. */ - vcpu->kvm = kvm; - vcpu->cpu_id = cpu_id; - vcpu->cpu_type = vcpu_init.target; - vcpu->cpu_compatible = target->compatible; - vcpu->is_running = true; -Why this change?
Quoted 2 lines
I will add a comment to describe why I moved this. During pvtime setup, in vcpu initialization I need to access the kvm structure from the vcpu.
Yeah, figured that out after I sent the email. Would you mind turning this change into a separate patch, something like "arm: Populate the VCPU struct before target->init()"? Please feel free to word the patch however you see fit.
Thanks,
Alex
Quoted 14 lines
Quoted 4 lines
Thanks,
AlexThanks for the review,
SebastianQuoted 6 lines
Quoted 6 lines
arm/kvm-cpu.c +0
-0if (kvm_cpu__configure_features (vcpu))die ("Unable to configure requested vcpu features" );Signature · 1 line
2.35.1.265.g69c8d7142f-goog