Skip to contentSkip to conversation

arm64: Basic SVE guest support

PATCH New Dave Martin 1 participant 4 messages Mar 20, 2019

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v2 cover arm64: Basic SVE guest support

This series, based on kvmtool master [1], contains development hacks to accompany the recently posted v6 SVE guest support series for the Linux kernel [2].

This series supersedes the previous posting [3], and is a clean(er) rewrite of the minimal support required for SVE, without the debug/ testing cruft.

Due to an ABI change, this series does not work with the older (v5) KVM series.

Testing and review welcome.

In particular, if people have a view on the --sve-vls parameter, I'd be interested to discuss what that should look like. See patch 3.

[1]
git://git.kernel.org/pub/scm/linux/kernel/git/will/kvmtool.git master
https://git.kernel.org/pub/scm/linux/kernel/git/will/kvmtool.git/log/

Specifically, commit c57e001a3efb ("arm: Auto-detect guest GIC type").

[2] [PATCH v5 00/26] KVM: arm64: SVE guest support
https://lists.cs.columbia.edu/pipermail/kvmarm/2019-March/thread.html

(Note, the subject line for this posting was incorrect. It should have
read: [PATCH v6 00/27] KVM: arm64: SVE guest support)

git://linux-arm.org/linux-dm.git sve-kvm/v6/head
http://linux-arm.org/git?p=linux-dm.git;a=shortlog;h=refs/heads/sve-kvm/v6/head

Dave Martin (3):
arm/arm64: [HACK] Sync kernel headers from sve-guest v6
arm64: Add basic SVE support
arm64: Add command-line option to set the guest's SVE vector lengths

 arm/aarch32/include/kvm/kvm-cpu-arch.h    |  5 ++
 arm/aarch64/include/asm/kvm.h             | 22 +++++++
 arm/aarch64/include/kvm/kvm-config-arch.h | 10 +++-
 arm/aarch64/include/kvm/kvm-cpu-arch.h    |  7 ++-
 arm/aarch64/kvm-cpu.c                     | 97 +++++++++++++++++++++++++++++++
 arm/include/arm-common/kvm-config-arch.h  |  2 +
 arm/kvm-cpu.c                             |  3 +
 include/linux/kvm.h                       |  5 ++
 8 files changed, 149 insertions(+), 2 deletions(-)
Signature · 1 line
2.1.4
#1 lore raw

PATCH v2 1/3 arm/arm64: [HACK] Sync kernel headers from sve-guest v6

This patch imports the necessary UAPI header definitions for building the SVE extensions for kvmtool.

This commit should be replaced by a header import from upstream Linux once the SVE kvm patches have been merged.

---
 arm/aarch64/include/asm/kvm.h | 22 ++++++++++++++++++++++
 include/linux/kvm.h           |  5 +++++
 2 files changed, 27 insertions(+)
arm/aarch64/include/asm/kvm.h +22 -0
index 97c3478..6963b7e 100644
@@ -102,6 +102,7 @@ struct kvm_regs {
 #define KVM_ARM_VCPU_EL1_32BIT		1 /* CPU running a 32bit VM */
 #define KVM_ARM_VCPU_PSCI_0_2		2 /* CPU uses PSCI v0.2 */
 #define KVM_ARM_VCPU_PMU_V3		3 /* Support guest PMUv3 */
+#define KVM_ARM_VCPU_SVE		4 /* enable SVE for this CPU */
 
 struct kvm_vcpu_init {
 	__u32 target;
@@ -226,6 +227,27 @@ struct kvm_vcpu_events {
 					 KVM_REG_ARM_FW | ((r) & 0xffff))
 #define KVM_REG_ARM_PSCI_VERSION	KVM_REG_ARM_FW_REG(0)
 
+/* SVE registers */
+#define KVM_REG_ARM64_SVE		(0x15 << KVM_REG_ARM_COPROC_SHIFT)
+
+/* Z- and P-regs occupy blocks at the following offsets within this range: */
+#define KVM_REG_ARM64_SVE_ZREG_BASE	0
+#define KVM_REG_ARM64_SVE_PREG_BASE	0x400
+
+#define KVM_REG_ARM64_SVE_ZREG(n, i)	(KVM_REG_ARM64 | KVM_REG_ARM64_SVE | \
+					 KVM_REG_ARM64_SVE_ZREG_BASE |	\
+					 KVM_REG_SIZE_U2048 |		\
+					 ((n) << 5) | (i))
+#define KVM_REG_ARM64_SVE_PREG(n, i)	(KVM_REG_ARM64 | KVM_REG_ARM64_SVE | \
+					 KVM_REG_ARM64_SVE_PREG_BASE |	\
+					 KVM_REG_SIZE_U256 |		\
+					 ((n) << 5) | (i))
+#define KVM_REG_ARM64_SVE_FFR(i)	KVM_REG_ARM64_SVE_PREG(16, i)
+
+/* Vector lengths pseudo-register: */
+#define KVM_REG_ARM64_SVE_VLS		(KVM_REG_ARM64 | KVM_REG_ARM64_SVE | \
+					 KVM_REG_SIZE_U512 | 0xffff)
+
 /* Device Control API: ARM VGIC */
 #define KVM_DEV_ARM_VGIC_GRP_ADDR	0
 #define KVM_DEV_ARM_VGIC_GRP_DIST_REGS	1
include/linux/kvm.h +5 -0
index 6d4ea4b..1d56444 100644
@@ -988,6 +988,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_VM_IPA_SIZE 165
 #define KVM_CAP_MANUAL_DIRTY_LOG_PROTECT 166
 #define KVM_CAP_HYPERV_CPUID 167
+#define KVM_CAP_ARM_SVE 168
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1145,6 +1146,7 @@ struct kvm_dirty_tlb {
 #define KVM_REG_SIZE_U256	0x0050000000000000ULL
 #define KVM_REG_SIZE_U512	0x0060000000000000ULL
 #define KVM_REG_SIZE_U1024	0x0070000000000000ULL
+#define KVM_REG_SIZE_U2048	0x0080000000000000ULL
 
 struct kvm_reg_list {
 	__u64 n; /* number of regs */
@@ -1440,6 +1442,9 @@ struct kvm_enc_region {
 /* Available with KVM_CAP_HYPERV_CPUID */
 #define KVM_GET_SUPPORTED_HV_CPUID _IOWR(KVMIO, 0xc1, struct kvm_cpuid2)
 
+/* Available with KVM_CAP_ARM_SVE */
+#define KVM_ARM_VCPU_FINALIZE	  _IOW(KVMIO,  0xc2, int)
+
 /* Secure Encrypted Virtualization command */
 enum sev_cmd_id {
 	/* Guest initialization commands */
Signature · 1 line
2.1.4
#2 lore raw

PATCH v2 2/3 arm64: Add basic SVE support

In reply to Dave Martin

This patch adds an --sve command line option to allow the Scalable Vector Extension to be enabled when creating a guest.

This requires use of the new KVM_ARM_VCPU_FINALIZE ioctl before the vcpu is runnable, so a new hook kvm_cpu__configure_features() is added to provide an appropiate place to do this work.

The kernel does not enable SVE by default, and for now kvmtool adopts the same policy: without --sve, SVE is not enabled for the guest even if the host supports it.

---
 arm/aarch32/include/kvm/kvm-cpu-arch.h    |  5 +++++
 arm/aarch64/include/kvm/kvm-config-arch.h |  4 +++-
 arm/aarch64/include/kvm/kvm-cpu-arch.h    |  7 ++++++-
 arm/aarch64/kvm-cpu.c                     | 19 +++++++++++++++++++
 arm/include/arm-common/kvm-config-arch.h  |  1 +
 arm/kvm-cpu.c                             |  3 +++
 6 files changed, 37 insertions(+), 2 deletions(-)
arm/aarch32/include/kvm/kvm-cpu-arch.h +5 -0
index d28ea67..04740d4 100644
@@ -13,4 +13,9 @@
 #define ARM_CPU_ID		0, 0, 0
 #define ARM_CPU_ID_MPIDR	5
 
+static inline int kvm_cpu__configure_features(struct kvm_cpu *vcpu)
+{
+	return 0;
+}
+
 #endif /* KVM__KVM_CPU_ARCH_H */
arm/aarch64/include/kvm/kvm-config-arch.h +3 -1
index 04be43d..50b7aae 100644
@@ -8,7 +8,9 @@
 			"Create PMUv3 device"),				\
 	OPT_U64('\0', "kaslr-seed", &(cfg)->kaslr_seed,			\
 			"Specify random seed for Kernel Address Space "	\
-			"Layout Randomization (KASLR)"),
+			"Layout Randomization (KASLR)"),		\
+	OPT_BOOLEAN('\0', "sve", &(cfg)->has_sve,			\
+			"Enable SVE for the guest"),
 
 #include "arm-common/kvm-config-arch.h"
 
arm/aarch64/include/kvm/kvm-cpu-arch.h +6 -1
index a9d8563..7f2bfb9 100644
@@ -8,13 +8,18 @@
 #define ARM_VCPU_FEATURE_FLAGS(kvm, cpuid)	{				\
 	[0] = ((!!(cpuid) << KVM_ARM_VCPU_POWER_OFF) |				\
 	       (!!(kvm)->cfg.arch.aarch32_guest << KVM_ARM_VCPU_EL1_32BIT) |	\
-	       (!!(kvm)->cfg.arch.has_pmuv3 << KVM_ARM_VCPU_PMU_V3))		\
+	       (!!(kvm)->cfg.arch.has_pmuv3 << KVM_ARM_VCPU_PMU_V3) |		\
+	       (!!(kvm)->cfg.arch.has_sve << KVM_ARM_VCPU_SVE))			\
 }
 
+
+
 #define ARM_MPIDR_HWID_BITMASK	0xFF00FFFFFFUL
 #define ARM_CPU_ID		3, 0, 0, 0
 #define ARM_CPU_ID_MPIDR	5
 #define ARM_CPU_CTRL		3, 0, 1, 0
 #define ARM_CPU_CTRL_SCTLR_EL1	0
 
+int kvm_cpu__configure_features(struct kvm_cpu *vcpu);
+
 #endif /* KVM__KVM_CPU_ARCH_H */
arm/aarch64/kvm-cpu.c +19 -0
index 0aaefaf..43eb69e 100644
@@ -128,6 +128,25 @@ static void reset_vcpu_aarch64(struct kvm_cpu *vcpu)
 	}
 }
 
+static int configure_sve(struct kvm_cpu *vcpu)
+{
+	int feature = KVM_ARM_VCPU_SVE;
+
+	if (ioctl(vcpu->vcpu_fd, KVM_ARM_VCPU_FINALIZE, &feature))
+		return -1;
+
+	return 0;
+}
+
+int kvm_cpu__configure_features(struct kvm_cpu *vcpu)
+{
+	if (vcpu->kvm->cfg.arch.has_sve)
+		if (configure_sve(vcpu))
+			return -1;
+
+	return 0;
+}
+
 void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
 {
 	if (vcpu->kvm->cfg.arch.aarch32_guest)
arm/include/arm-common/kvm-config-arch.h +1 -0
index 5734c46..bdab680 100644
@@ -12,6 +12,7 @@ struct kvm_config_arch {
 	u64		kaslr_seed;
 	enum irqchip_type irqchip;
 	u64		fw_addr;
+	bool		has_sve;
 };
 
 int irqchip_parser(const struct option *opt, const char *arg, int unset);
arm/kvm-cpu.c +3 -0
index 7780251..91d5241 100644
@@ -122,6 +122,9 @@ struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
 	vcpu->cpu_compatible	= target->compatible;
 	vcpu->is_running	= true;
 
+	if (kvm_cpu__configure_features(vcpu))
+		die("Unable to configure requested vcpu features");
+
 	return vcpu;
 }
 
Signature · 1 line
2.1.4
#3 lore raw

PATCH v2 3/3 arm64: Add command-line option to set the guest's SVE vector lengths

In reply to Dave Martin

In order to support use cases such as migration, it may be important in some situations to restrict the set of SVE vector lengths available to the guest. It can also be usefule to observe the behaviour of guest OSes with different vector lengths.

To enable testing and experimentation for such configurations, this patch adds a command-line option to allow setting of the set of vector lengths to be made available to the guest.

For now, the setting is global: no means is offered to configure individual guest vcpus independently of each other.

---

It wasn't clear to me what the most user-friendly form for the
--sve-vls parameter would be. Possibilities include:

	--sve-max-vl=<n>	# all host vector lengths not exceeding <n>
	--sve-vls=16,32,64	# i.e., byte ("vl") units, not 128-bit
	--sve-vls=1-2,4		# i.e., similar syntax to taskset -c etc.
	--sve-vls=16-32,64	# combination of the above
	--sve-vqs=1,2,4		# for more consistent terminology

In the end, to avoid premature overengineering, I went with the easiest choice. (--sve-vls=1,2,4)

There's also the question of how to specify different features and configurations per-vcpu. I don't currently attempt this, but it would be useful for KVM testing and development. I think that any solution to this shouldn't be SVE-specific: we should aim for a common approach that can be reused for other features too.

---
 arm/aarch64/include/kvm/kvm-config-arch.h |  8 +++-
 arm/aarch64/kvm-cpu.c                     | 80 ++++++++++++++++++++++++++++++-
 arm/include/arm-common/kvm-config-arch.h  |  1 +
 3 files changed, 87 insertions(+), 2 deletions(-)
arm/aarch64/include/kvm/kvm-config-arch.h +7 -1
index 50b7aae..1eb8cfc 100644
@@ -1,6 +1,8 @@
 #ifndef KVM__KVM_CONFIG_ARCH_H
 #define KVM__KVM_CONFIG_ARCH_H
 
+int sve_vls_parser(const struct option *opt, const char *arg, int unset);
+
 #define ARM_OPT_ARCH_RUN(cfg)						\
 	OPT_BOOLEAN('\0', "aarch32", &(cfg)->aarch32_guest,		\
 			"Run AArch32 guest"),				\
@@ -10,7 +12,11 @@
 			"Specify random seed for Kernel Address Space "	\
 			"Layout Randomization (KASLR)"),		\
 	OPT_BOOLEAN('\0', "sve", &(cfg)->has_sve,			\
-			"Enable SVE for the guest"),
+			"Enable SVE for the guest"),			\
+	OPT_CALLBACK('\0', "sve-vls", &(cfg)->sve_vqs,			\
+		     "comma-separated list of vector lengths, in 128-bit units", \
+		     "Set of vector lengths to enable for the guest",	\
+		     sve_vls_parser, NULL),
 
 #include "arm-common/kvm-config-arch.h"
 
arm/aarch64/kvm-cpu.c +79 -1

108 lines not on this page · raw message

arm/include/arm-common/kvm-config-arch.h +1 -0
index bdab680..ca11e55 100644
@@ -13,6 +13,7 @@ struct kvm_config_arch {
 	enum irqchip_type irqchip;
 	u64		fw_addr;
 	bool		has_sve;
+	u64		*sve_vqs;
 };
 
 int irqchip_parser(const struct option *opt, const char *arg, int unset);
Signature · 1 line
2.1.4